def get_waveforms():
events = get_events()[::-1]
client = Client('GFZ')
stream_raw = Stream()
stream = RFStream()
coords = inventory.get_coordinates(seedid[:-1] + 'Z')
for i, event in enumerate(events):
t = event.preferred_origin().time
args = seedid.split('.') + [t + 4.9 * 60, t + 14.1 * 60]
s = client.get_waveforms(*args)
s.trim(t+5*60, t+14*60)
s.decimate(int(round(s[0].stats.sampling_rate)) // 5, no_filter=True)
stream_raw.extend(s)
if i in (0, 2, 4):
s = s.copy()
stats = rfstats(station=coords, event=event, dist_range=(20, 95))
if stats is None:
continue
s.trim(stats.onset - 25, stats.onset + 75)
stats = obj2stats(station=coords, event=event)
s = RFStream(s)
for tr in s:
tr.stats.update(stats)
stream.extend(s)
stream_raw.write(wavname, 'MSEED')
stream.write(wavname2, 'SAC')
def test_moveout_vs_XY(self):
stream = RFStream(read())[:1]
for tr in stream:
tr.stats.slowness = 10.
tr.stats.onset = tr.stats.starttime + 20.643
tr.stats.phase = 'P'
stream.decimate(10)
N = len(stream[0])
t = np.linspace(0, 20 * np.pi, N)
stream[0].data = np.sin(t) * np.exp(-0.04 * t)
stream[0].stats.slowness = 4.0
stream1 = stream.copy()
stream2 = stream.copy()
stream3 = stream.copy()
stream3[0].stats.slowness = 9.0
stream4 = stream3.copy()
stream5 = stream.copy()
stream6 = stream.copy()
stream7 = stream.copy()
stream8 = stream.copy()
stream9 = stream.copy()
stream10 = stream.copy()
stream1.moveout()
stream3.moveout()
stream5.moveout(phase='Ppps')
stream7.moveout(phase='Ppss')
stream9.moveout(phase='Psss')
# stream2._moveout_xy()
# print(repr(stream2[0].data))
# stream4._moveout_xy()
# print(repr(stream6[0].data))
# stream6._moveout_xy(phase='Ppps')
# print(repr(stream6[0].data))
# stream8._moveout_xy(phase='Ppss')
# print(repr(stream8[0].data))
# stream10._moveout_xy(phase='Psss')
# print(repr(stream10[0].data))
stream2[0].data = XY_PSMOUT_REF4
stream4[0].data = XY_PSMOUT_REF9
stream6[0].data = XY_PPPSMOUT_REF4
stream8[0].data = XY_PPSSMOUT_REF4
stream8[0].data = XY_PSSSMOUT_REF4
np.testing.assert_array_almost_equal(stream1[0].data,
stream2[0].data,
decimal=2)
np.testing.assert_array_almost_equal(stream3[0].data,
stream4[0].data,
decimal=2)
np.testing.assert_array_almost_equal(stream5[0].data,
stream6[0].data,
decimal=2)
np.testing.assert_array_almost_equal(stream7[0].data,
stream8[0].data,
decimal=2)
np.testing.assert_array_almost_equal(stream9[0].data,
stream10[0].data,
decimal=2)
def profile(stream, boxes, crs=None):
"""
Stack traces in stream by piercing point coordinates in defined boxes.
:param stream: stream with pre-calculated piercing point coordinates
:param boxes: boxes created with `get_profile_boxes()`
:param crs: cartopy projection (default: AzimuthalEquidistant)
:return: profile stream
"""
stack = {}
for tr in stream:
ppoint = (tr.stats.pp_latitude, tr.stats.pp_longitude)
box = _find_box(ppoint, boxes, crs=crs)
if box is None:
continue
pos = box['pos']
comp = tr.stats.channel[-1]
key = (pos, comp)
if key not in stack:
header = {
'box_pos': pos,
'box_length': box['length'],
'box_latitude': box['latlon'][0],
'box_longitude': box['latlon'][1],
'profile_latitude': boxes[0]['profile']['latlon'][0],
'profile_longitude': boxes[0]['profile']['latlon'][1],
'profile_azimuth': boxes[0]['profile']['azimuth'],
'profile_length': boxes[0]['profile']['length'],
'num': 1,
'sampling_rate': tr.stats.sampling_rate,
'channel': '??' + comp
}
for entry in ('slowness', 'phase', 'moveout', 'processing'):
if entry in tr.stats:
header[entry] = tr.stats[entry]
stack[key] = tr2 = tr.__class__(data=tr.data, header=header)
if 'onset' in tr.stats:
onset = tr.stats.onset - tr.stats.starttime
tr2.stats.onset = tr2.stats.starttime + onset
else:
tr2 = stack[key]
tr2.data = tr2.data + tr.data
tr2.stats.num += 1
for tr2 in stack.values():
tr2.data = tr2.data / tr2.stats.num
if hasattr(stream, 'iterable'): # support tqdm objects
cls = stream.iterable.__class__
else:
cls = stream.__class__
try:
profile = cls(traces=stack.values())
except TypeError: # stream can be an iterator
from rf import RFStream
profile = RFStream(traces=stack.values())
profile.sort(['channel', 'box_pos'])
profile.type = 'profile'
return profile
def profile(stream, boxes, crs=None):
"""
Stack traces in stream by piercing point coordinates in defined boxes.
:param stream: stream with pre-calculated piercing point coordinates
:param boxes: boxes created with `get_profile_boxes()`
:param crs: cartopy projection (default: AzimuthalEquidistant)
:return: profile stream
"""
stack = {}
for tr in stream:
ppoint = (tr.stats.pp_latitude, tr.stats.pp_longitude)
box = _find_box(ppoint, boxes, crs=crs)
if box is None:
continue
pos = box['pos']
comp = tr.stats.channel[-1]
key = (pos, comp)
if key not in stack:
header = {'box_pos': pos,
'box_length': box['length'],
'box_latitude': box['latlon'][0],
'box_longitude': box['latlon'][1],
'profile_latitude': boxes[0]['profile']['latlon'][0],
'profile_longitude': boxes[0]['profile']['latlon'][1],
'profile_azimuth': boxes[0]['profile']['azimuth'],
'profile_length': boxes[0]['profile']['length'],
'num': 1,
'sampling_rate': tr.stats.sampling_rate,
'channel': '??' + comp}
for entry in ('slowness', 'phase', 'moveout', 'processing'):
if entry in tr.stats:
header[entry] = tr.stats[entry]
stack[key] = tr2 = tr.__class__(data=tr.data, header=header)
if 'onset' in tr.stats:
onset = tr.stats.onset - tr.stats.starttime
tr2.stats.onset = tr2.stats.starttime + onset
else:
tr2 = stack[key]
tr2.data = tr2.data + tr.data
tr2.stats.num += 1
for tr2 in stack.values():
tr2.data = tr2.data / tr2.stats.num
if hasattr(stream, 'iterable'): # support tqdm objects
cls = stream.iterable.__class__
else:
cls = stream.__class__
try:
profile = cls(traces=stack.values())
except TypeError: # stream can be an iterator
from rf import RFStream
profile = RFStream(traces=stack.values())
profile.sort(['channel', 'box_pos'])
profile.type = 'profile'
return profile
def compute_ned_stacked_rf(ned):
# Compute RF from NetworkEventDataset and return R component
rf_all = RFStream()
for _sta, evid, stream in ned:
rf_3ch = transform_stream_to_rf(evid, RFStream(stream),
DEFAULT_CONFIG_FILTERING,
DEFAULT_CONFIG_PROCESSING)
if rf_3ch is None:
continue
rf_all += rf_3ch.select(component='R')
# end for
rf_stacked = rf_all.stack()
return rf_stacked, rf_all
def _run_single_station(db_evid, angles, config_filtering, config_processing):
"""
Internal processing function for running sequence of candidate angles
over a single station.
:param db_evid: Dictionary of event streams (3-channel ZNE) keyed by event ID. \
Best obtained using class NetworkEventDataset
:type db_evid: sortedcontainers.SortedDict or similar dict-like
:param angles: Sequence of candidate correction angles to try (degrees)
:type angles: Iterable(float)
:param config_filtering: Waveform filtering options for RF processing
:type config_filtering: dict
:param config_processing: RF processing options
:type config_processing: dict
:return: Amplitude metric as a function of angle. Same length as angles array.
:rtype: list(float)
"""
ampls = []
for correction in angles:
rf_stream_all = RFStream()
for evid, stream in db_evid.items():
stream_rot = copy.deepcopy(stream)
for tr in stream_rot:
tr.stats.back_azimuth += correction
while tr.stats.back_azimuth < 0:
tr.stats.back_azimuth += 360
while tr.stats.back_azimuth >= 360:
tr.stats.back_azimuth -= 360
# end for
rf_3ch = transform_stream_to_rf(evid, RFStream(stream_rot),
config_filtering,
config_processing)
if rf_3ch is None:
continue
rf_stream_all += rf_3ch
# end for
if len(rf_stream_all) > 0:
rf_stream_R = rf_stream_all.select(component='R')
rf_stream_R.trim2(-5, 5, reftime='onset')
rf_stream_R.detrend('linear')
rf_stream_R.taper(0.1)
R_stack = rf_stream_R.stack().trim2(-1, 1, reftime='onset')[0].data
ampl_mean = np.mean(R_stack)
else:
ampl_mean = np.nan
# endif
ampls.append(ampl_mean)
# end for
return ampls
class SimpleModelTestCase(unittest.TestCase):
def setUp(self):
self.stream = RFStream(read())
self.stream._write_test_header()
self.model = load_model()
def test_ppoint(self):
xy_plat_P = 50.29670878 # 200km sppier
xy_plon_P = -97.24776461 # 200km sppier
xy_plat_S = 50.29862202 # 200km pspier
xy_plon_S = -98.96394212 # 200km pspier
st = self.stream[0].stats
self.model.ppoint(st, 200, phase='P')
self.assertLess(abs((st.plat - xy_plat_P) /
(st.plat - st.station_latitude)), 1) # very big
self.assertLess(abs((st.plon - xy_plon_P) /
(st.plon - st.station_longitude)), 0.05)
# print 'station lon ', st.station_longitude
# print 'xy lon P ', xy_plon_P
# print 'model lon P ', st.plon
# print
# print 'station lat ', st.station_latitude
# print 'xy lat P ', xy_plat_P
# print 'model lat P ', st.plat
# print
self.model.ppoint(st, 200, phase='S')
self.assertLess(abs((st.plat - xy_plat_S) /
(st.plat - st.station_latitude)), 1) # very big
self.assertLess(abs((st.plon - xy_plon_S) /
(st.plon - st.station_longitude)), 0.05)
# print 'station lon ', st.station_longitude
# print 'xy lon S ', xy_plon_S
# print 'model lon S ', st.plon
# print
# print 'station lat ', st.station_latitude
# print 'xy lat S ', xy_plat_S
# print 'model lat S ', st.plat
def test_moveout(self):
self.model.moveout(self.stream)
i = 20
for tr in self.stream:
tr.stats.distance = i
i = i + 50
self.model.moveout(self.stream)
class SimpleModelTestCase(unittest.TestCase):
def setUp(self):
self.stream = RFStream(read())
self.stream._write_test_header()
def test_ppoint(self):
xy_plat_P = 50.29670878 # 200km sppier
xy_plon_P = -97.24776461 #200km sppier
xy_plat_S = 50.29862202 # 200km pspier
xy_plon_S = -98.96394212 #200km pspier
st = self.stream[0].stats
ppoint(self.stream, 200, phase='P')
self.assertLess(abs((st.plat - xy_plat_P) /
(st.plat - st.station_latitude)), 1) # very big
self.assertLess(abs((st.plon - xy_plon_P) /
(st.plon - st.station_longitude)), 0.05)
# print 'station lon ', st.station_longitude
# print 'xy lon P ', xy_plon_P
# print 'model lon P ', st.plon
# print
# print 'station lat ', st.station_latitude
# print 'xy lat P ', xy_plat_P
# print 'model lat P ', st.plat
# print
ppoint(self.stream, 200, phase='S')
self.assertLess(abs((st.plat - xy_plat_S) /
(st.plat - st.station_latitude)), 1) # very big
self.assertLess(abs((st.plon - xy_plon_S) /
(st.plon - st.station_longitude)), 0.05)
# print 'station lon ', st.station_longitude
# print 'xy lon S ', xy_plon_S
# print 'model lon S ', st.plon
# print
# print 'station lat ', st.station_latitude
# print 'xy lat S ', xy_plat_S
# print 'model lat S ', st.plat
def test_moveout(self):
# st1 = self.stream.copy()
# moveout(self.stream)
# st2 = self.stream
# st = st1[:1] + st2[:1]
# st = st1 + st2
# st.plot(automerge=False)
moveout(self.stream)
def test_moveout_vs_XY(self):
stream = RFStream(read())[:1]
for tr in stream:
tr.stats.slowness = 10.
tr.stats.onset = tr.stats.starttime + 20.643
tr.stats.phase = 'P'
stream.decimate(10)
N = len(stream[0])
t = np.linspace(0, 20*np.pi, N)
stream[0].data = np.sin(t)*np.exp(-0.04*t)
stream[0].stats.slowness = 4.0
stream1 = stream.copy()
stream2 = stream.copy()
stream3 = stream.copy()
stream3[0].stats.slowness = 9.0
stream4 = stream3.copy()
stream5 = stream.copy()
stream6 = stream.copy()
stream7 = stream.copy()
stream8 = stream.copy()
stream9 = stream.copy()
stream10 = stream.copy()
stream1.moveout()
stream3.moveout()
stream5.moveout(phase='Ppps')
stream7.moveout(phase='Ppss')
stream9.moveout(phase='Psss')
# stream2._moveout_xy()
# print(repr(stream2[0].data))
# stream4._moveout_xy()
# print(repr(stream6[0].data))
# stream6._moveout_xy(phase='Ppps')
# print(repr(stream6[0].data))
# stream8._moveout_xy(phase='Ppss')
# print(repr(stream8[0].data))
# stream10._moveout_xy(phase='Psss')
# print(repr(stream10[0].data))
stream2[0].data = XY_PSMOUT_REF4
stream4[0].data = XY_PSMOUT_REF9
stream6[0].data = XY_PPPSMOUT_REF4
stream8[0].data = XY_PPSSMOUT_REF4
stream8[0].data = XY_PSSSMOUT_REF4
np.testing.assert_array_almost_equal(stream1[0].data, stream2[0].data,
decimal=2)
np.testing.assert_array_almost_equal(stream3[0].data, stream4[0].data,
decimal=2)
np.testing.assert_array_almost_equal(stream5[0].data, stream6[0].data,
decimal=2)
np.testing.assert_array_almost_equal(stream7[0].data, stream8[0].data,
decimal=2)
np.testing.assert_array_almost_equal(stream9[0].data, stream10[0].data,
decimal=2)
def do_rf(stream3c):
stream3c = stream3c.detrend('linear').interpolate(100)
stream3c = stream3c.taper(0.01)
stream3c = stream3c.filter('bandpass',
freqmin=0.03,
freqmax=0.80,
corners=2,
zerophase=True)
if len(stream3c) != 3:
return RFStream()
a1 = stream3c[0].stats['asdf']
a2 = stream3c[1].stats['asdf']
a3 = stream3c[2].stats['asdf']
stream3c[0].stats['asdf'] = []
stream3c[1].stats['asdf'] = []
stream3c[2].stats['asdf'] = []
# LQT receiver functions are default
# stream3c.rf()
# ZRT receiver functions must be specified
stream3c.rf(rotate='NE->RT')
# stream3c.rf(rotate='NE->RT',deconvolve='freq',gauss=2.0)
'''
Note the parameters of gaussian pulse and its width where
Value of "a" | Frequency (hz) at which G(f) = 0.1 | Approximate Pulse Width (s)
10 4.8 0.50
5 2.4 0.75
2.5 1.2 1.00
1.25 0.6 1.50
1.0 0.5 1.67 (5/3)
0.625 0.3 2.10
0.5 0.24 2.36
0.4 0.2 2.64
0.2 0.1 3.73
'''
amax = {'amax': np.amax(stream3c[0].data)}
stream3c[0].stats['asdf'] = a1
stream3c[0].stats.update(amax)
# stream3c[0].filter('bandpass', freqmin=0.03, freqmax=1.00, corners=2, zerophase=True)
# stream3c[0].data=stream3c[0].data*(amax['amax']/np.amax(stream3c[0].data))
amax = {'amax': np.amax(stream3c[1].data)}
stream3c[1].stats['asdf'] = a2
stream3c[1].stats.update(amax)
# stream3c[1].filter('bandpass', freqmin=0.03, freqmax=1.00, corners=2, zerophase=True)
# stream3c[1].data=stream3c[0].data*(amax['amax']/np.amax(stream3c[0].data))
amax = {'amax': np.amax(stream3c[2].data)}
stream3c[2].stats['asdf'] = a3
stream3c[2].stats.update(amax)
# stream3c[2].filter('bandpass', freqmin=0.03, freqmax=1.00, corners=2, zerophase=True)
# stream3c[2].data=stream3c[0].data*(amax['amax']/np.amax(stream3c[0].data))
stream3c.trim2(-25, 75, 'onset')
# print np.max(stream3c[0].data),np.max(stream3c[1].data),np.max(stream3c[2].data)
return stream3c
def test_io_header(self):
stream = RFStream(read())[:1]
for tr in stream:
tr.stats.location = '11'
tr.stats.pop('response', None)
write_test_header(stream)
test_io_header(self, stream)
def retrieve_waveform(client,net,stn,t1,t2,stats_dict=None,cha="BHE,BHN,BHZ",attach_response=False,loc="",pharr=None, phasenm = 'P'):
try:
st = client.get_waveforms(net, stn, loc, cha, t1, t2,attach_response=attach_response)
except:
return False
# print("Retrieving")
if phasenm == 'P':
# filter_traces(st,lenphase=int(t2-t1))
filter_traces_rf(st,pharr = pharr)
elif phasenm == 'SKS':
filter_traces_sks(st,pharr = pharr)
if len(st) != 3:
# print(f"All three components not available: {len(st)}")
return False
if stats_dict:
dist, baz, _ = gps2dist_azimuth(stats_dict['station_latitude'],stats_dict['station_longitude'],stats_dict['event_latitude'],stats_dict['event_longitude'])
for tr in st:
for key, value in stats_dict.items():
tr.stats[key] = value
tr.stats['distance'] = dist / 1000 / DEG2KM
tr.stats['back_azimuth'] = baz
st.merge()
# if all 3 components present and no gap or overlap in data
if len(st) == 3 and not any(isinstance(tr.data, np.ma.masked_array) for tr in st):
# print(f"Stream obtained {len(st)}")
return RFStream(st)
elif not any(isinstance(tr.data, np.ma.masked_array) for tr in st):
# print("--------> There's a gap/overlap in the data")
return False
def do_rf(stream3c):
stream3c.detrend('linear').resample(100)
stream3c.taper(0.01)
stream3c.filter('bandpass',
freqmin=0.01,
freqmax=15,
corners=2,
zerophase=True)
if len(stream3c) != 3:
return RFStream()
a1 = stream3c[0].stats['asdf']
a2 = stream3c[1].stats['asdf']
a3 = stream3c[2].stats['asdf']
stream3c[0].stats['asdf'] = []
stream3c[1].stats['asdf'] = []
stream3c[2].stats['asdf'] = []
# LQT receiver functions are default
# stream3c.rf()
# ZRT receiver functions must be specified
stream3c.rf(rotate='NE->RT')
stream3c[0].stats['asdf'] = a1
stream3c[1].stats['asdf'] = a2
stream3c[2].stats['asdf'] = a3
stream3c.trim2(-25, 75, 'onset')
return stream3c
def test_io_header(self):
def test_io_format(format):
stream1 = stream.copy()
fname = self.temp + '_IO_FORMAT.' + format.upper()
if format == 'sh':
format = 'q'
fname = self.temp + '.QHD'
stream1.write(fname, format.upper())
stream2 = RFStream(stream=read(fname))
st1 = stream1[0].stats
st2 = stream2[0].stats
for head in HEADERS:
self.assertAlmostEqual(st1[head], st2[head], 4, msg=head)
stream = RFStream(stream=read())[:1]
stream._write_test_header()
for format in FORMATHEADERS:
test_io_format(format)
def get_waveforms():
events = get_events()[::-1]
client = Client('GFZ')
stream_raw = Stream()
stream = RFStream()
coords = inventory.get_coordinates(seedid[:-1] + 'Z')
for i, event in enumerate(events):
t = event.preferred_origin().time
args = seedid.split('.') + [t + 4.9 * 60, t + 14.1 * 60]
s = client.get_waveforms(*args)
s.trim(t + 5 * 60, t + 14 * 60)
s.decimate(int(round(s[0].stats.sampling_rate)) // 5, no_filter=True)
stream_raw.extend(s)
if i in (0, 2, 4):
s = s.copy()
stats = rfstats(station=coords, event=event, dist_range=(20, 95))
if stats is None:
continue
s.trim(stats.onset - 25, stats.onset + 75)
stats = obj2stats(station=coords, event=event)
s = RFStream(s)
for tr in s:
tr.stats.update(stats)
stream.extend(s)
stream_raw.write(wavname, 'MSEED')
stream.write(wavname2, 'SAC')
def async_write(rfstream_queue, outfile_name, max_buffered=100):
"""Monitors asynchronous queue for data, removes from queue to buffer, then
flushes buffer intermittently and when queue termination signal is put.
When None is received on the queue, this is taken as the signal to terminate
monitoring the queue.
:param rfstream_queue: Queue into which RFStreams are pushed for writing to file.
:type rfstream_queue: multiprocessing.Manager.Queue
:param outfile_name: Name of file into which queued RFStream results are periodically written.
:type outfile_name: str or Path
:param max_buffered: Maximum number of RFStreams to buffer before flushing to file, defaults to 100
:type max_buffered: int, optional
"""
buffered_streams = []
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.info("Starting async write thread")
first_write = True
while True:
# Passing None into the queue is taken as signal to flush buffer and terminate thread
rfstream = rfstream_queue.get()
terminating = (rfstream is None)
if not terminating:
buffered_streams.append(rfstream)
else:
logger.info("Flushing result buffer...")
if len(buffered_streams) >= max_buffered or terminating:
stream = RFStream()
for rf in buffered_streams:
stream.extend(rf)
if first_write:
mode = 'w'
first_write = False
else:
mode = 'a'
stream.write(outfile_name, 'H5', mode=mode)
logger.info("Flushed {} streams to output file {}".format(
len(buffered_streams), outfile_name))
while buffered_streams:
buffered_streams.pop()
rfstream_queue.task_done()
if terminating:
rfstream_queue.task_done()
break
logger.info("Terminating async write thread")
def __iter__(self):
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.info("Scanning station groups from file {}".format(
self.h5_filename))
with self._open_source_file() as f:
wf_data = f['waveforms']
num_stations = len(wf_data)
count = 0
event_count = 0
for station_id in wf_data:
count += 1
logger.info("Station {} {}/{}".format(station_id, count,
num_stations))
station_data = wf_data[station_id]
station_stream3c = []
for event_time in station_data:
event_traces = station_data[event_time]
# if len(event_traces) != self.num_components:
# logging.warning("Incorrect number of traces ({}) for stn {} event {}, skipping"
# .format(len(event_traces), station_id, event_time))
# continue
traces = []
for trace_id in event_traces:
trace = dataset2trace(event_traces[trace_id])
traces.append(trace)
event_count += 1
station_stream3c.append(RFStream(traces=traces).sort())
# end for
# Yield the results with 3-channel trace triplets grouped together in RFStream instances.
yield station_id, station_stream3c
# end for
# end with
logger.info("Yielded {} event traces to process".format(event_count))
import os.path
# import matplotlib.pyplot as plt
# import numpy as np
from rf import read_rf, RFStream
from rf import IterMultipleComponents
# from rf.imaging import plot_profile_map
# from rf.profile import profile
from tqdm import tqdm
data = read_rf('DATA/7X-event_waveforms_for_rf.h5', 'H5')
# exclude bad stations
inc_set = list(set([tr.stats.inclination for tr in data]))
data_filtered = RFStream([
tr for tr in data if tr.stats.inclination in inc_set
and tr.stats.station not in ['MIJ2', 'MIL2']
])
stream = RFStream()
for stream3c in tqdm(IterMultipleComponents(data, 'onset', 3)):
stream3c.detrend('linear').resample(100)
stream3c.taper(0.01)
stream3c.filter('bandpass', freqmin=0.01, freqmax=15)
if len(stream3c) != 3:
continue
a1 = stream3c[0].stats['asdf']
a2 = stream3c[1].stats['asdf']
a3 = stream3c[2].stats['asdf']
stream3c[0].stats['asdf'] = []
stream3c[1].stats['asdf'] = []
stream3c[2].stats['asdf'] = []
in_streamfile = 'data/7X-rf_profile_data-15deg.h5'
out_streamfile = 'data/7X-rf_profile_data-15deg-out.h5'
st = read_rf(in_streamfile, 'H5')
stz = [tr for tr in st if tr.stats.channel.endswith('Z')]
stn = [tr for tr in st if tr.stats.channel.endswith('N')]
ste = [tr for tr in st if tr.stats.channel.endswith('E')]
output_from_z = []
for trz, trn, tre in zip(stz, stn, ste):
band, margin, upper = extract_filter_params(trz)
if band and margin and upper:
for tr in [trz, trn, tre]:
clean_trace(tr,
tr.stats.starttime,
tr.stats.endtime,
freqmin=band[0],
freqmax=band[1])
print('Plot the cleaned trace here')
else:
print(
'None of filters produced an acceptable p-pick. discarding the 3 traces whose z-comp has stats: '
+ str(trz.stats))
stz.remove(trz)
stn.remove(trn)
ste.remove(tre)
st_out = stz + stn + ste
st_out_rf = RFStream(st_out)
st_out_rf.write(out_streamfile, 'H5')
import matplotlib.pyplot as plt
import numpy as np
from obspy import read_inventory, read_events, UTCDateTime as UTC
from obspy.clients.fdsn import Client
from rf import read_rf, RFStream
from rf import get_profile_boxes, iter_event_data, IterMultipleComponents
from rf.imaging import plot_profile_map
from rf.profile import profile
from tqdm import tqdm
data = read_rf('data/7X-rf_profile_data-15deg.h5', 'H5')
data_cleaned = read_rf('data/7X-rf_profile_data-15deg-out.h5', 'H5')
inc_set = list(set([tr.stats.inclination for tr in data_cleaned]))
data_filtered = RFStream([
tr for tr in data if tr.stats.inclination in inc_set
and tr.stats.station not in ['MIJ2', 'MIL2']
])
stream = RFStream()
for stream3c in tqdm(IterMultipleComponents(data_filtered, 'onset', 3)):
stream3c.filter('bandpass', freqmin=0.1, freqmax=1)
stream3c.trim2(-25, 75, 'onset')
if len(stream3c) != 3:
continue
stream3c.rf()
stream3c.moveout()
stream.extend(stream3c)
stream.write('data/7X-rf_profile_rfs_0.1Hz_1Hz', 'H5')
ppoints = stream.ppoints(70)
boxes = get_profile_boxes((-18.4, 139.1),
135,
np.linspace(0, 440, 80),
return st
# ---+----------Main---------------------------------
if __name__ == '__main__':
# we use centre of Australia to calculate radius and gather events from 15 to 90 degrees
lonlat = [133.88, -23.69]
# Change parameters below
data = os.path.join('DATA', '')
invfile = data + '7X-inventory.xml'
datafile = data + '7X-event_waveforms_for_rf.h5'
start_time = '2009-12-01 00:00:00'
end_time = '2011-04-01 00:00:00'
inventory = read_inventory(invfile)
# ----------------- End ----------------------
catalog = get_events(lonlat, UTC(start_time), UTC(end_time))
stream = RFStream()
with tqdm() as pbar:
for s in iter_event_data(catalog, inventory, custom_get_waveforms, pbar=pbar):
for trace in s:
stream.extend(s)
stream.write(datafile, 'H5')
'starttime': UTC('2010-01-01'),
'endtime': UTC('2011-01-01'),
'latitude': lonlat[1],
'longitude': lonlat[0],
'minradius': 30,
'maxradius': 90,
'minmagnitude': 5.5,
'maxmagnitude': 6.5
}
catalog = client.get_events(**kwargs)
catalog.write(catfile, 'QUAKEML')
catalog = read_events(catfile)
if not os.path.exists(datafile):
client = Client('GFZ')
stream = RFStream()
with tqdm() as pbar:
for s in iter_event_data(catalog,
inventory,
client.get_waveforms,
pbar=pbar):
stream.extend(s)
stream.write(datafile, 'H5')
if not os.path.exists(rffile):
data = read_rf(datafile, 'H5')
stream = RFStream()
for stream3c in tqdm(IterMultipleComponents(data, 'onset', 3)):
stream3c.filter('bandpass', freqmin=0.5, freqmax=2)
stream3c.trim2(-25, 75, 'onset')
if len(stream3c) != 3:
def download_data(self,
catalogtxtloc,
datafileloc,
tot_evnt_stns,
rem_evnts,
plot_stations=True,
plot_events=True,
dest_map="./",
locations=[""]):
self.logger.info(f"Total data files to download: {tot_evnt_stns}")
rem_dl = rem_evnts
succ_dl, num_try = 0, 0
rf_stalons, sks_stalons = [], []
rf_stalats, sks_stalats = [], []
rf_staNetNames, sks_staNetNames = [], []
all_stns_df = pd.read_csv(self.inventorytxtfile, sep="|")
all_sta_lats = all_stns_df['Latitude'].values
all_sta_lons = all_stns_df['Longitude'].values
all_sta_nms = all_stns_df['Station'].values
all_sta_nets = all_stns_df['#Network'].values
sta_str_list = []
#Retrive waveform data for the events
for slat, slon, stn, net in zip(all_sta_lats, all_sta_lons,
all_sta_nms, all_sta_nets):
sta_str = f"{net}-{stn}-{slon}-{slat}"
if sta_str in sta_str_list:
continue
else:
sta_str_list.append(sta_str)
catfile = catalogtxtloc + f"{net}-{stn}-events-info-{self.method}.txt"
cattxtnew = catalogtxtloc + f"{net}-{stn}-events-info-available-{self.method}.txt"
if self.method == 'RF':
print("\n")
self.logger.info(
f"Searching and downloading data for {self.method}; {net}-{stn}"
)
rfdatafile = datafileloc + f"{net}-{stn}-{str(inpRFdict['filenames']['data_rf_suffix'])}.h5"
# rfdatafile = datafileloc+f"{net}-{stn}-{str(inpRF.loc['data_rf_suffix','VALUES'])}.h5"
# rfdatafile = datafileloc+f"{net}-{stn}-{str(inpRF.loc['data_rf_suffix','VALUES'])}.h5"
if os.path.exists(catfile) and not os.path.exists(
rfdatafile) and tot_evnt_stns > 0:
stream = RFStream()
df = pd.read_csv(catfile, sep=",")
# df = pd.read_csv(catfile,delimiter="\||,", names=['evtime','evlat','evlon','evdp','evmg','evmgtp'],header=None,engine="python")
evmg = df['evmg'].values
evmgtp = ["Mww" for val in df['evmg']]
# evmg = [float(val.split()[0]) for val in df['evmg']]
# evmgtp = [str(val.split()[1]) for val in df['evmg']]
fcat = open(cattxtnew, 'w')
for evtime, evdp, elat, elon, em, emt in zip(
df['evtime'], df['evdp'], df['evlat'], df['evlon'],
evmg, evmgtp):
rem_dl -= 1
num_try += 1
strm, res, msg = multi_download(
self.client,
self.inv,
net,
stn,
slat,
slon,
elat,
elon,
evdp,
evtime,
em,
emt,
fcat,
stalons=rf_stalons,
stalats=rf_stalats,
staNetNames=rf_staNetNames,
phase='P',
locations=locations)
if res:
succ_dl += 1
if not msg:
self.logger.info(
f"Event: {evtime}; rem: {rem_dl}/{tot_evnt_stns}; dl: {succ_dl}/{num_try}"
)
else:
self.logger.info(
f"{msg}; rem: {rem_dl}/{tot_evnt_stns}; dl: {succ_dl}/{num_try}"
)
if strm:
stream.extend(strm)
if not len(stream):
self.logger.warning(f"No data {rfdatafile}")
stream.write(rfdatafile, 'H5')
fcat.close()
### Event map plot
# cattxtnew = catalogtxtloc+f'{net}-{stn}-events-info-rf.txt'
if os.path.exists(
cattxtnew
) and plot_events and not os.path.exists(
f"{net}-{stn}-{str(inpRFdict['filenames']['events_map_suffix'])}.png"
):
# if os.path.exists(cattxtnew) and plot_events and not os.path.exists(f"{net}-{stn}-{str(inpRF.loc['events_map_suffix','VALUES'])}.png"):
df = pd.read_csv(cattxtnew,
delimiter="\||,",
names=[
'evtime', 'evlat', 'evlon', 'evdp',
'evmg', 'client'
],
header=None,
engine="python")
if df.shape[0]:
evmg = [float(val.split()[0]) for val in df['evmg']]
event_plot_name = f"{net}-{stn}-{str(inpRFdict['filenames']['events_map_suffix'])}"
# event_plot_name=f"{net}-{stn}-{str(inpRF.loc['events_map_suffix','VALUES'])}"
if not os.path.exists(dest_map + event_plot_name +
f".{self.fig_frmt}"):
self.logger.info(f"Plotting events map " +
event_plot_name +
f".{self.fig_frmt}")
events_map(evlons=df['evlon'],
evlats=df['evlat'],
evmgs=evmg,
evdps=df['evdp'],
stns_lon=slon,
stns_lat=slat,
destination=dest_map,
figfrmt=self.fig_frmt,
clon=slon,
outname=f'{event_plot_name}')
if self.method == 'SKS':
print("\n")
self.logger.info(
f"Searching and downloading data for {self.method}; {net}-{stn}"
)
sksdatafile = datafileloc + f"{net}-{stn}-{str(inpSKSdict['filenames']['data_sks_suffix'])}.h5"
if os.path.exists(catfile) and not os.path.exists(
sksdatafile) and tot_evnt_stns > 0:
self.logger.info("Reading events catalog file")
stream = RFStream()
df = pd.read_csv(catfile, sep=",")
# df = pd.read_csv(catfile,delimiter="\||,", names=['evtime','evlat','evlon','evdp','evmg','evmgtp'],header=None,engine="python")
evmg = df['evmg'].values
evmgtp = ["Mww" for val in df['evmg']]
# evmg = [float(val.split()[0]) for val in df['evmg']]
# evmgtp = [str(val.split()[1]) for val in df['evmg']]
# cattxtnew = catalogtxtloc+f'{net}-{stn}-events-info-sks.txt'
fcat = open(cattxtnew, 'w')
for i, evtime, evdp, elat, elon, em, emt in zip(
range(len(df['evtime'])), df['evtime'], df['evdp'],
df['evlat'], df['evlon'], evmg, evmgtp):
rem_dl -= 1
num_try += 1
# self.logger.info(f"Event: {evtime}; rem: {rem_dl}/{tot_evnt_stns}; dl: {succ_dl}/{num_try}")
strm, res, msg = multi_download(
self.client,
self.inv,
net,
stn,
slat,
slon,
elat,
elon,
evdp,
evtime,
em,
emt,
fcat,
stalons=sks_stalons,
stalats=sks_stalats,
staNetNames=sks_staNetNames,
phase='SKS',
locations=locations)
if not msg:
self.logger.info(
f"Event: {evtime}; rem: {rem_dl}/{tot_evnt_stns}; dl: {succ_dl}/{num_try}"
)
else:
self.logger.info(
f"{msg}; rem: {rem_dl}/{tot_evnt_stns}; dl: {succ_dl}/{num_try}"
)
if strm:
stream.extend(strm)
if res:
succ_dl += 1
if not len(stream):
self.logger.warning(f"No data {sksdatafile}")
stream.write(sksdatafile, 'H5')
fcat.close()
else:
if os.path.exists(catfile):
self.logger.info(f"catalog {catfile} exists!")
else:
self.logger.info(f"catalog {catfile} does not exist!")
if not os.path.exists(sksdatafile):
self.logger.info(
f"datafile {sksdatafile} does not exist!")
if tot_evnt_stns > 0:
self.logger.info(
f"Total files to download {tot_evnt_stns}")
### Event map plot
# cattxtnew = catalogtxtloc+f'{net}-{stn}-events-info-sks.txt'
if os.path.exists(
cattxtnew
) and plot_events and not os.path.exists(
f"{net}-{stn}-{str(inpSKSdict['filenames']['events_map_suffix'])}.png"
):
df = pd.read_csv(cattxtnew,
delimiter="\||,",
names=[
'evtime', 'evlat', 'evlon', 'evdp',
'evmg', 'client'
],
header=None,
engine="python")
if df.shape[0]:
evmg = [float(val.split()[0]) for val in df['evmg']]
event_plot_name = f"{net}-{stn}-{str(inpSKSdict['filenames']['events_map_suffix'])}"
if not os.path.exists(dest_map + event_plot_name +
f".{self.fig_frmt}"):
self.logger.info(f"Plotting events map " +
event_plot_name +
f".{self.fig_frmt}")
events_map(evlons=df['evlon'],
evlats=df['evlat'],
evmgs=evmg,
evdps=df['evdp'],
stns_lon=slon,
stns_lat=slat,
destination=dest_map,
figfrmt=self.fig_frmt,
clon=slon,
outname=f'{net}-{stn}-SKS')
## plot station map for all the stations for which the data has been successfully retrieved
if plot_stations and self.method == 'RF' and len(rf_stalons):
print("\n")
self.logger.info("Plotting station map for RF")
map = plot_merc(resolution='h',
llcrnrlon=self.minlongitude - 1,
llcrnrlat=self.minlatitude - 1,
urcrnrlon=self.maxlongitude + 1,
urcrnrlat=self.maxlatitude + 1,
topo=True)
station_map(map,
stns_lon=rf_stalons,
stns_lat=rf_stalats,
stns_name=rf_staNetNames,
figname=str(
inpRFdict['filenames']['retr_station_prefix']),
destination=dest_map,
figfrmt=self.fig_frmt)
# station_map(map, stns_lon=rf_stalons, stns_lat=rf_stalats,stns_name= rf_staNetNames,figname=str(inpRF.loc['retr_station_prefix','VALUES']), destination=dest_map,figfrmt=self.fig_frmt)
if plot_stations and self.method == 'SKS' and len(sks_stalons):
print("\n")
self.logger.info("Plotting station map for SKS")
map = plot_merc(resolution='h',
llcrnrlon=self.minlongitude - 1,
llcrnrlat=self.minlatitude - 1,
urcrnrlon=self.maxlongitude + 1,
urcrnrlat=self.maxlatitude + 1,
topo=True)
station_map(map,
stns_lon=sks_stalons,
stns_lat=sks_stalats,
stns_name=sks_staNetNames,
figname=str(
inpSKSdict['filenames']['retr_station_prefix']),
destination=dest_map,
figfrmt=self.fig_frmt)
## Write the retrieved station catalog
if self.method == 'RF':
write_station_file(self.inventorytxtfile,
rf_staNetNames,
outfile=catalogtxtloc +
str(inpRFdict['filenames']['retr_stations']))
# write_station_file(self.inventorytxtfile,rf_staNetNames,outfile=catalogtxtloc+str(inpRF.loc['retr_stations','VALUES']))
elif self.method == 'SKS':
write_station_file(self.inventorytxtfile,
sks_staNetNames,
outfile=catalogtxtloc +
str(inpSKSdict['filenames']['retr_stations']))
def iter_event_data(events, inventory, get_waveforms, phase='P',
request_window=None, pad=10, pbar=None, **kwargs):
"""
Return iterator yielding three component streams per station and event.
:param events: list of events or `~obspy.core.event.Catalog` instance
:param inventory: `~obspy.core.inventory.inventory.Inventory` instance
with station and channel information
:param get_waveforms: Function returning the data. It has to take the
arguments network, station, location, channel, starttime, endtime.
:param phase: Considered phase, e.g. 'P', 'S', 'PP'
:type request_window: tuple (start, end)
:param request_window: requested time window around the onset of the phase
:param float pad: add specified time in seconds to request window and
trim afterwards again
:param pbar: tqdm_ instance for displaying a progressbar
:param kwargs: all other kwargs are passed to `~rf.rfstream.rfstats()`
:return: three component streams with raw data
Example usage with progressbar::
from tqdm import tqdm
from rf.util import iter_event_data
with tqdm() as t:
for stream3c in iter_event_data(*args, pbar=t):
do_something(stream3c)
.. _tqdm: https://pypi.python.org/pypi/tqdm
"""
logger = logging.getLogger(__name__)
from rf.rfstream import rfstats, RFStream
method = phase[-1].upper()
if request_window is None:
request_window = (-50, 150) if method == 'P' else (-100, 50)
# else:
# print(f"\nRequested window is {request_window}")
stations = _get_stations(inventory)
# print(f"Available stations are {stations}")
if pbar is not None:
pbar.total = len(events) * len(stations)
for event, seedid in itertools.product(events, stations):
if pbar is not None:
pbar.update(1)
origin_time = (event.preferred_origin() or event.origins[0])['time']
# print(f"Origin time is {origin_time}")
try:
args = (seedid[:-1] + stations[seedid], origin_time)
coords = inventory.get_coordinates(*args)
# print(f"Station coords are {coords}")
except Exception as exception: # station not available at that time
logger.error("Station not available at given time", exc_info=True)
continue
try:
stats = rfstats(station=coords, event=event, phase=phase, **kwargs)
except:
logger.warning(f"Unable to read for: {event}")
pass
if not stats:
continue
net, sta, loc, cha = seedid.split('.')
starttime = stats.onset + request_window[0]
endtime = stats.onset + request_window[1]
kws = {'network': net, 'station': sta, 'location': loc,
'channel': cha, 'starttime': starttime - pad,
'endtime': endtime + pad}
try:
stream = get_waveforms(**kws)
# print(f"stream obtained {len(stream)}")
except: # no data available
# logger.warning(f"No data available for {event}")
continue
stream.trim(starttime, endtime)
stream.merge()
# print(f"length of the stream is {len(stream)}")
if len(stream) != 3:
from warnings import warn
warn(f'Need 3 component seismograms. {len(stream)} components '
'detected for event {event.resource_id}, station {seedid}.')
logger.warning(f'Need 3 component seismograms. {len(stream)} components '
'detected for event {event.resource_id}, station {seedid}.')
continue
if any(isinstance(tr.data, np.ma.masked_array) for tr in stream):
from warnings import warn
logger.warning(f'Gaps or overlaps detected for event {event.resource_id}, station {seedid}.')
continue
for tr in stream:
tr.stats.update(stats)
yield RFStream(stream)
def __iter__(self):
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.info("Scanning jobs metadata from file {}".format(self.h5_filename))
with self._open_source_file() as f:
wf_data = f['waveforms']
num_stations = len(wf_data)
count = 0
event_count = 0
create_event_id = False
first_loop = True
for station_id in wf_data:
count += 1
logger.info("Station {} {}/{}".format(station_id, count, num_stations))
station_data = wf_data[station_id]
for event_time in station_data:
event_traces = station_data[event_time]
if not event_traces:
continue
if first_loop:
first_loop = False
tmp = list(event_traces.keys())[0]
create_event_id = ('event_id' not in event_traces[tmp].attrs)
traces = []
for trace_id in event_traces:
trace = dataset2trace(event_traces[trace_id])
traces.append(trace)
stream = RFStream(traces=traces)
if len(stream) != self.num_components and self.channel_pattern is not None:
for ch_mask in self.channel_pattern.split(','):
_stream = stream.select(channel=ch_mask)
logging.info("Tried channel mask {}, got {} channels".format(ch_mask, len(_stream)))
if len(_stream) == self.num_components:
stream = _stream
break
# end for
# end if
if len(stream) != self.num_components:
logging.warning("Incorrect number of traces ({}) for stn {} event {}, skipping"
.format(len(stream), station_id, event_time))
continue
# end if
# Force order of traces to ZNE ordering.
stream.traces = sorted(stream.traces, key=zne_order)
# Strongly assert expected ordering of traces. This must be respected so that
# RF normalization works properly.
assert stream.traces[0].stats.channel[-1] == 'Z'
assert stream.traces[1].stats.channel[-1] == 'N'
assert stream.traces[2].stats.channel[-1] == 'E'
event_count += 1
if create_event_id:
event_id = event_count
else:
event_id = traces[0].stats.event_id
assert np.all([(tr.stats.event_id == event_id) for tr in traces])
# end if
yield station_id, event_id, event_time, stream
# end for
# end for
# end with
logger.info("Yielded {} event traces to process".format(event_count))
def compute_rf(dataRFfileloc):
logger = logging.getLogger(__name__)
all_rfdatafile = glob.glob(
dataRFfileloc +
f"*-{str(inpRFdict['filenames']['data_rf_suffix'])}.h5")
for jj, rfdatafile in enumerate(all_rfdatafile):
network = rfdatafile.split("-")[0]
station = rfdatafile.split("-")[1]
rffile = f"{network}-{station}-{str(inpRFdict['filenames']['rf_compute_data_suffix'])}.h5"
datatmp = read_rf(rfdatafile, 'H5')
if not os.path.exists(rffile):
logger.info(
f"--> Computing RF for {rfdatafile}, {jj+1}/{len(all_rfdatafile)}"
)
data = read_rf(rfdatafile, 'H5')
stream = RFStream()
for stream3c in tqdm.tqdm(IterMultipleComponents(data, 'onset',
3)):
if len(stream3c) != 3:
continue
## check if the length of all three traces are equal
lenphase = 100
for tr in stream3c:
lentr = tr.stats.npts
lengt = tr.stats.sampling_rate * lenphase
if lentr != lengt:
if tr.stats.sampling_rate < 20:
logger.warning(
f"Sampling rate too low: {tr.stats.sampling_rate}, required >= 20Hz"
)
stream3c.remove(tr)
continue
elif tr.stats.sampling_rate >= 20:
if tr.stats.sampling_rate % 20 == 0:
factor = int(tr.stats.sampling_rate / 20)
tr.decimate(factor,
strict_length=False,
no_filter=True)
if tr.stats.npts > tr.stats.sampling_rate * lenphase:
t = tr.stats.starttime
tr.trim(
t, t + lenphase -
(1 / tr.stats.sampling_rate))
continue
else:
tr.resample(20.0)
if tr.stats.npts > tr.stats.sampling_rate * lenphase:
t = tr.stats.starttime
tr.trim(
t, t + lenphase -
(1 / tr.stats.sampling_rate))
continue
else:
pass
test_npts = []
for tr in stream3c:
lentr = tr.stats.npts
test_npts.append(lentr)
if len(set(test_npts)) > 1:
continue
stream3c.filter(
'bandpass',
freqmin=float(inpRFdict['rf_filter_settings']['minfreq']),
freqmax=float(inpRFdict['rf_filter_settings']['maxfreq']))
try:
stream3c.rf()
except Exception as e:
logger.warning("Problem applying rf method", exc_info=True)
stream3c.moveout()
stream.extend(stream3c)
stream.write(rffile, 'H5')
else:
# logger.info(f"--> {rffile} already exists!, {jj}/{len(all_rfdatafile)}")
logger.info(
f"--> Verifying RF computation {jj+1}/{len(all_rfdatafile)}")
stream3c.trim2(-25, 75, 'onset')
# print np.max(stream3c[0].data),np.max(stream3c[1].data),np.max(stream3c[2].data)
return stream3c
print "Lets start the show..."
#data = read_rf('DATA/7X-event_waveforms_for_rf.h5', 'H5')
data = read_rf('DATA/7X-MA12.h5', 'H5')
print "Data in..."
'''
# we can exclude bad stations
inc_set = list(set([tr.stats.inclination for tr in data]))
data_filtered = RFStream([tr for tr in data if tr.stats.inclination in inc_set and tr.stats.station not in ['MIJ2', 'MIL2']])
'''
stream = RFStream()
rf_streams = Parallel(n_jobs=-1,
verbose=1)(map(delayed(do_rf),
IterMultipleComponents(data, 'onset', 3)))
for i, rf in enumerate(rf_streams):
event_id = {'event_id': 0}
event_id['event_id'] = i
for tr in rf:
tr.stats.update(event_id)
stream.extend(rf)
stream.write('DATA/7X-MA12-rf_zrt', 'H5')
print "No worries, mate..."
def setUp(self):
self.stream = RFStream(read())
self.stream._write_test_header()
def test_simple(self):
stream = RFStream(stream=read())
stream._write_test_header()
stream.rf()
stream.moveout()
stream.ppoint(50)
def setUp(self):
self.stream = RFStream(read())
self.stream._write_test_header()
self.model = load_model()