def test_rt_gaussian_filter(self):
from am_signal import gaussian_filter
data_trace = self.data_trace.copy()
gauss5, tshift = gaussian_filter(1.0, 5.0, 0.01)
rt_trace = RtTrace()
rt_single = RtTrace()
for rtt in [rt_trace, rt_single]:
rtt.registerRtProcess('convolve', conv_signal=gauss5)
rt_single.append(data_trace, gap_overlap_check=True)
for tr in self.traces:
# pre-apply inversed time-shift before appending data
tr.stats.starttime -= tshift
rt_trace.append(tr, gap_overlap_check=True)
# test the waveforms are the same
diff = self.data_trace.copy()
diff.data = rt_trace.data - rt_single.data
self.assertAlmostEquals(np.mean(np.abs(diff)), 0.0)
# test the time-shifts
starttime_diff = rt_single.stats.starttime - self.data_trace.stats.starttime
self.assertAlmostEquals(starttime_diff, 0.0)
def test_rt_kurt_grad(self):
win = 3.0
data_trace = self.data_trace.copy()
sigma = float(np.std(data_trace.data))
fact = 1 / sigma
rt_trace = RtTrace()
rt_trace_single = RtTrace()
for rtt in [rt_trace, rt_trace_single]:
rtt.registerRtProcess('scale', factor=fact)
rtt.registerRtProcess('kurtosis', win=win)
rtt.registerRtProcess('boxcar', width=50)
rtt.registerRtProcess('differentiate')
rtt.registerRtProcess('neg_to_zero')
rt_trace_single.append(data_trace)
for tr in self.traces:
rt_trace.append(tr, gap_overlap_check=True)
diff = self.data_trace.copy()
diff.data = rt_trace_single.data - rt_trace.data
self.assertAlmostEquals(np.mean(np.abs(diff)), 0.0, 5)
def test_ne(self):
"""
Testing __ne__ method.
"""
tr = Trace()
tr2 = RtTrace()
tr3 = RtTrace()
# RtTrace should never be equal with Trace objects
self.assertNotEqual(tr2, tr)
self.assertTrue(tr2.__ne__(tr))
self.assertFalse(tr2 != tr3)
self.assertFalse(tr2.__ne__(tr3))
def test_append_sanity_checks(self):
"""
Testing sanity checks of append method.
"""
rtr = RtTrace()
ftr = Trace(data=np.array([0, 1]))
# sanity checks need something already appended
rtr.append(ftr)
# 1 - differing ID
tr = Trace(header={'network': 'xyz'})
self.assertRaises(TypeError, rtr.append, tr)
tr = Trace(header={'station': 'xyz'})
self.assertRaises(TypeError, rtr.append, tr)
tr = Trace(header={'location': 'xy'})
self.assertRaises(TypeError, rtr.append, tr)
tr = Trace(header={'channel': 'xyz'})
self.assertRaises(TypeError, rtr.append, tr)
# 2 - sample rate
tr = Trace(header={'sampling_rate': 100.0})
self.assertRaises(TypeError, rtr.append, tr)
tr = Trace(header={'delta': 0.25})
self.assertRaises(TypeError, rtr.append, tr)
# 3 - calibration factor
tr = Trace(header={'calib': 100.0})
self.assertRaises(TypeError, rtr.append, tr)
# 4 - data type
tr = Trace(data=np.array([0.0, 1.1]))
self.assertRaises(TypeError, rtr.append, tr)
# 5 - only Trace objects are allowed
self.assertRaises(TypeError, rtr.append, 1)
self.assertRaises(TypeError, rtr.append, "2323")
def test_rt_variance(self):
win = 10
data_trace = self.data_trace.copy()
rt_single = RtTrace()
rt_trace = RtTrace()
rt_trace.registerRtProcess('variance', win=win)
rt_single.registerRtProcess('variance', win=win)
for tr in self.traces:
rt_trace.append(tr, gap_overlap_check=True)
rt_single.append(data_trace, gap_overlap_check=True)
assert_array_almost_equal(rt_single, rt_trace)
def test_sw_kurtosis(self):
win = 3.0
data_trace = self.data_trace.copy()
rt_trace = RtTrace()
rt_single = RtTrace()
rt_trace.registerRtProcess('sw_kurtosis', win=win)
rt_single.registerRtProcess('sw_kurtosis', win=win)
for tr in self.traces:
rt_trace.append(tr, gap_overlap_check=True)
rt_single.append(data_trace)
diff = self.data_trace.copy()
diff.data = rt_trace.data - rt_single.data
self.assertAlmostEquals(np.mean(np.abs(diff)), 0.0)
def test_append_not_float32(self):
"""
Test for not using float32.
"""
tr = read()[0]
tr.data = np.require(tr.data, dtype=native_str('>f4'))
traces = tr / 3
rtr = RtTrace()
for trace in traces:
rtr.append(trace)
def test_rt_mean(self):
win = 0.05
data_trace = self.data_trace.copy()
rt_single = RtTrace()
rt_trace = RtTrace()
rt_trace.registerRtProcess('mean', win=win)
rt_single.registerRtProcess('mean', win=win)
for tr in self.traces:
rt_trace.append(tr, gap_overlap_check=True)
rt_single.append(data_trace, gap_overlap_check=True)
newtr = self.data_trace.copy()
newtr.data = newtr.data - rt_trace.data
assert_array_almost_equal(rt_single, rt_trace)
self.assertAlmostEqual(np.mean(newtr.data), 0.0, 0)
def test_kwin_bank(self):
win_list = [1.0, 3.0, 9.0]
n_win = len(win_list)
data_trace = self.data_trace.copy()
sigma = float(np.std(data_trace.data))
fact = 1 / sigma
# One RtTrace for processing before the kurtosis
rt_trace = RtTrace()
rt_trace.registerRtProcess('scale', factor=fact)
# One RtTrace per kurtosis window
kurt_traces = []
for i in xrange(n_win):
rtt = RtTrace()
rtt.registerRtProcess('kurtosis', win=win_list[i])
kurt_traces.append(rtt)
# One RrTrace for post-processing the max kurtosis window
max_kurt = RtTrace()
max_kurt.registerRtProcess('differentiate')
max_kurt.registerRtProcess('neg_to_zero')
for tr in self.traces:
# prepare memory for kurtosis
kurt_tr = tr.copy()
# do initial processing
proc_trace = rt_trace.append(tr, gap_overlap_check=True)
kurt_output = []
for i in xrange(n_win):
# pass output of initial processing to the kwin bank
ko = kurt_traces[i].append(proc_trace, gap_overlap_check=True)
# append the output to the kurt_output list
kurt_output.append(ko.data)
# stack the output of the kwin bank and find maximum
kurt_stack = np.vstack(tuple(kurt_output))
kurt_tr.data = np.max(kurt_stack, axis=0)
# append to the max_kurt RtTrace for post-processing
max_kurt.append(kurt_tr)
def test_missing_or_wrong_argument_in_rt_process(self):
"""
Tests handling of missing/wrong arguments.
"""
trace = Trace(np.arange(100))
# 1- function scale needs no additional arguments
rt_trace = RtTrace()
rt_trace.register_rt_process('scale')
rt_trace.append(trace)
# adding arbitrary arguments should fail
rt_trace = RtTrace()
rt_trace.register_rt_process('scale', muh='maeh')
self.assertRaises(TypeError, rt_trace.append, trace)
# 2- function tauc has one required argument
rt_trace = RtTrace()
rt_trace.register_rt_process('tauc', width=10)
rt_trace.append(trace)
# wrong argument should fail
rt_trace = RtTrace()
rt_trace.register_rt_process('tauc', xyz='xyz')
self.assertRaises(TypeError, rt_trace.append, trace)
# missing argument width should raise an exception
rt_trace = RtTrace()
rt_trace.register_rt_process('tauc')
self.assertRaises(TypeError, rt_trace.append, trace)
# adding arbitrary arguments should fail
rt_trace = RtTrace()
rt_trace.register_rt_process('tauc', width=20, notexistingoption=True)
self.assertRaises(TypeError, rt_trace.append, trace)
def test_append_overlap(self):
"""
Appending overlapping traces should raise a UserWarning/TypeError
"""
rtr = RtTrace()
tr = Trace(data=np.array([0, 1]))
rtr.append(tr)
# this raises UserWarning
with warnings.catch_warnings(record=True):
warnings.simplefilter('error', UserWarning)
self.assertRaises(UserWarning, rtr.append, tr)
# append with gap_overlap_check=True will raise a TypeError
self.assertRaises(TypeError, rtr.append, tr, gap_overlap_check=True)
def test_rt_offset(self):
offset = 500
rt_trace = RtTrace()
rt_trace.registerRtProcess('offset', offset=offset)
for tr in self.traces:
rt_trace.append(tr, gap_overlap_check=True)
diff = self.data_trace.copy()
diff.data = rt_trace.data - self.data_trace.data
self.assertAlmostEquals(np.mean(np.abs(diff)), offset)
def test_rt_kurtosis_dec(self):
win = 5.0
data_trace = self.data_trace_filt.copy()
data_trace_dec = self.data_trace_filt.copy()
# no need to filter as we're using a pre-filtered trace
data_trace_dec.decimate(5, no_filter=True)
rt_trace = RtTrace()
rt_dec = RtTrace()
rt_trace.registerRtProcess('kurtosis', win=win)
rt_dec.registerRtProcess('kurtosis', win=win)
rt_trace.append(data_trace, gap_overlap_check=True)
rt_dec.append(data_trace_dec, gap_overlap_check=True)
newtr = rt_trace.copy()
newtr.decimate(5, no_filter=True)
#assert_array_almost_equal(rt_dec.data, newtr.data, 0)
diff = (np.max(rt_dec.data) - np.max(newtr.data)) / np.max(rt_dec.data)
self.assertAlmostEquals(np.abs(diff), 0.0, 2)
def test_append_gap(self):
"""
Appending a traces with a time gap should raise a UserWarning/TypeError
"""
rtr = RtTrace()
tr = Trace(data=np.array([0, 1]))
tr2 = Trace(data=np.array([5, 6]))
tr2.stats.starttime = tr.stats.starttime + 10
rtr.append(tr)
# this raises UserWarning
with warnings.catch_warnings(record=True):
warnings.simplefilter('error', UserWarning)
self.assertRaises(UserWarning, rtr.append, tr2)
# append with gap_overlap_check=True will raise a TypeError
self.assertRaises(TypeError, rtr.append, tr2, gap_overlap_check=True)
def test_rt_neg_to_zero(self):
data_trace = self.data_trace.copy()
max_val = np.max(data_trace.data)
rt_trace = RtTrace()
rt_trace.registerRtProcess('neg_to_zero')
for tr in self.traces:
rt_trace.append(tr, gap_overlap_check=True)
max_val_test = np.max(rt_trace.data)
min_val_test = np.min(rt_trace.data)
self.assertEqual(max_val, max_val_test)
self.assertEqual(0.0, min_val_test)
def test_copy(self):
"""
Testing copy of RtTrace object.
"""
rtr = RtTrace()
rtr.copy()
# register predefined function
rtr.registerRtProcess('integrate', test=1, muh='maeh')
rtr.copy()
# register ObsPy function call
rtr.registerRtProcess(filter.bandpass, freqmin=0, freqmax=1, df=0.1)
rtr.copy()
# register NumPy function call
rtr.registerRtProcess(np.square)
rtr.copy()
def test_rt_scale(self):
data_trace = self.data_trace.copy()
fact = 1 / np.std(data_trace.data)
data_trace.data *= fact
rt_trace = RtTrace()
rt_trace.registerRtProcess('scale', factor=fact)
for tr in self.traces:
rt_trace.append(tr, gap_overlap_check=True)
diff = self.data_trace.copy()
diff.data = rt_trace.data - data_trace.data
self.assertAlmostEquals(np.mean(np.abs(diff)), 0.0)
def _run_rt_process(self, process_list, max_length=None):
"""
Helper function to create a RtTrace, register all given process
functions and run the real time processing.
"""
# assemble real time trace
self.rt_trace = RtTrace(max_length=max_length)
for (process, options) in process_list:
self.rt_trace.register_rt_process(process, **options)
# append packet data to RtTrace
self.rt_appended_traces = []
for trace in self.orig_trace_chunks:
# process single trace
result = self.rt_trace.append(trace, gap_overlap_check=True)
# add to list of appended traces
self.rt_appended_traces.append(result)
def test_register_rt_process(self):
"""
Testing register_rt_process method.
"""
tr = RtTrace()
# 1 - function call
tr.register_rt_process(np.abs)
self.assertEqual(tr.processing, [(np.abs, {}, None)])
# 2 - predefined RT processing algorithm
tr.register_rt_process('integrate', test=1, muh='maeh')
self.assertEqual(tr.processing[1][0], 'integrate')
self.assertEqual(tr.processing[1][1], {'test': 1, 'muh': 'maeh'})
self.assertTrue(isinstance(tr.processing[1][2][0], RtMemory))
# 3 - contained name of predefined RT processing algorithm
tr.register_rt_process('in')
self.assertEqual(tr.processing[2][0], 'integrate')
tr.register_rt_process('integ')
self.assertEqual(tr.processing[3][0], 'integrate')
tr.register_rt_process('integr')
self.assertEqual(tr.processing[4][0], 'integrate')
# 4 - unknown functions
self.assertRaises(NotImplementedError, tr.register_rt_process,
'integrate2')
self.assertRaises(NotImplementedError, tr.register_rt_process, 'xyz')
# 5 - module instead of function
self.assertRaises(NotImplementedError, tr.register_rt_process, np)
# check number off all processing steps within RtTrace
self.assertEqual(len(tr.processing), 5)
# check tr.stats.processing
self.assertEqual(len(tr.stats.processing), 5)
self.assertTrue(tr.stats.processing[0].startswith("realtime_process"))
self.assertIn('absolute', tr.stats.processing[0])
for i in range(1, 5):
self.assertIn('integrate', tr.stats.processing[i])
# check kwargs
self.assertIn("maeh", tr.stats.processing[1])
def test_rt_kurtosis(self):
win = 3.0
data_trace = self.data_trace.copy()
sigma = float(np.std(data_trace.data))
fact = 1 / sigma
dt = data_trace.stats.delta
C1 = dt / float(win)
x = data_trace.data
ktrace = data_trace.copy()
ktrace.data = rec_kurtosis(x * fact, C1)
rt_trace = RtTrace()
rt_trace.registerRtProcess('scale', factor=fact)
rt_trace.registerRtProcess('kurtosis', win=win)
for tr in self.traces:
rt_trace.append(tr, gap_overlap_check=True)
diff = self.data_trace.copy()
diff.data = rt_trace.data - ktrace.data
self.assertAlmostEquals(np.mean(np.abs(diff)), 0.0)
def __init__(self, waveloc_options):
"""
Initialize from a set of travel-times as hdf5 files
"""
wo = waveloc_options
# initialize the travel-times
#############################
ttimes_fnames = glob.glob(wo.ttimes_glob)
# get basic lengths
f = h5py.File(ttimes_fnames[0], 'r')
# copy the x, y, z data over
self.x = np.array(f['x'][:])
self.y = np.array(f['y'][:])
self.z = np.array(f['z'][:])
f.close()
# read the files
ttimes_list = []
self.sta_list = []
for fname in ttimes_fnames:
f = h5py.File(fname, 'r')
# update the list of ttimes
ttimes_list.append(np.array(f['ttimes']))
sta = f['ttimes'].attrs['station']
f.close()
# update the dictionary of station names
self.sta_list.append(sta)
# stack the ttimes into a numpy array
self.ttimes_matrix = np.vstack(ttimes_list)
(self.nsta, self.npts) = self.ttimes_matrix.shape
# initialize the RtTrace(s)
##########################
max_length = wo.opdict['max_length']
self.safety_margin = wo.opdict['safety_margin']
self.dt = wo.opdict['dt']
# need a RtTrace per station
self.obs_rt_list = [RtTrace() for sta in self.sta_list]
# register pre-processing
self._register_preprocessing(wo)
# need nsta streams for each point we test (nsta x npts)
# for shifted waveforms
self.point_rt_list=[[RtTrace(max_length=max_length) \
for ista in xrange(self.nsta)] for ip in xrange(self.npts)]
# register processing of point-streams here
for sta_list in self.point_rt_list:
for rtt in sta_list:
# This is where we would scale for distance (given pre-calculated
# distances from each point to every station)
rtt.registerRtProcess('scale', factor=1.0)
# need npts streams to store the point-stacks
self.stack_list = [
RtTrace(max_length=max_length) for ip in xrange(self.npts)
]
# register stack procesing here
for rtt in self.stack_list:
# This is where we would add or lower weights if we wanted to
rtt.registerRtProcess('scale', factor=1.0)
# need 4 output streams (max, x, y, z)
self.max_out = RtTrace()
self.x_out = RtTrace()
self.y_out = RtTrace()
self.z_out = RtTrace()
if not wo.is_syn:
self.max_out.registerRtProcess('boxcar', width=50)
# need a list of common start-times
self.last_common_end_stack = [
UTCDateTime(1970, 1, 1) for i in xrange(self.npts)
]
self.last_common_end_max = UTCDateTime(1970, 1, 1)
def save_wave(self):
# Fetch a wave from Ring 0
wave = self.ring2buff.get_wave(0)
# if wave is empty return
if wave == {}:
return
# Lets try to buffer with python dictionaries and obspy
name = wave["station"] + '.' + wave["channel"] + '.' + wave[
"network"] + '.' + wave["location"]
if name in self.wave_buffer:
# Determine max samples for buffer
max_samp = wave["samprate"] * 60 * self.minutes
# Create a header:
wavestats = Stats()
wavestats.station = wave["station"]
wavestats.network = wave["network"]
wavestats.channel = wave["channel"]
wavestats.location = wave["location"]
wavestats.sampling_rate = wave["samprate"]
wavestats.starttime = UTCDateTime(wave['startt'])
# Create a trace
wavetrace = Trace(header=wavestats)
wavetrace.data = wave["data"]
# Try to append data to buffer, if gap shutdown.
try:
self.wave_buffer[name].append(wavetrace,
gap_overlap_check=True)
except TypeError as err:
logger.warning(err)
self.runs = False
except:
raise
self.runs = False
# Debug data
if self.debug:
logger.info("Station Channel combo is in buffer:")
logger.info(name)
logger.info("Size:")
logger.info(self.wave_buffer[name].count())
logger.debug("Data:")
logger.debug(self.wave_buffer[name])
else:
# First instance of data in buffer, create a header:
wavestats = Stats()
wavestats.station = wave["station"]
wavestats.network = wave["network"]
wavestats.channel = wave["channel"]
wavestats.location = wave["location"]
wavestats.sampling_rate = wave["samprate"]
wavestats.starttime = UTCDateTime(wave['startt'])
# Create a trace
wavetrace = Trace(header=wavestats)
wavetrace.data = wave["data"]
# Create a RTTrace
rttrace = RtTrace(int(self.minutes * 60))
self.wave_buffer[name] = rttrace
# Append data
self.wave_buffer[name].append(wavetrace, gap_overlap_check=True)
# Debug data
if self.debug:
logger.info("First instance of station/channel:")
logger.info(name)
logger.info("Size:")
logger.info(self.wave_buffer[name].count())
logger.debug("Data:")
logger.debug(self.wave_buffer[name])