def _check_available_data(archive, arc_type, day):
"""
Function to check what stations are available in the archive for a given \
day.
:type archive: str
:param archive: The archive source
:type arc_type: str
:param arc_type: The type of archive, can be:
:type day: datetime.date
:param day: Date to retrieve data for
:returns: list of tuples of (station, channel) as available.
..note:: Currently the seishub options are untested.
"""
from obspy import read, UTCDateTime
available_stations = []
if arc_type.lower() == 'day_vols':
wavefiles = glob.glob(os.path.join(archive, day.strftime('%Y'),
day.strftime('%j.01'), '*'))
for wavefile in wavefiles:
header = read(wavfile, headonly=True)
available_stations.append((header[0].stats.station,
header[0].stats.channel))
elif arc_type.lower() == 'seishub':
from obspy.clients.seishub import Client
client = Client(archive)
st = client.get_previews(starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + 86400)
for tr in st:
available_stations.append((tr.stats.station, tr.stats.channel))
elif arc_type.lower() == 'fdsn':
from obspy.clients.fdsn import Client
client = Client(archive)
inventory = client.get_stations(starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + 86400,
level='channel')
for network in inventory:
for station in network:
for channel in station:
available_stations.append((station.code,
channel.code))
return available_stations
def _check_available_data(archive, arc_type, day):
"""
Function to check what stations are available in the archive for a given \
day.
:type archive: str
:param archive: The archive source
:type arc_type: str
:param arc_type: The type of archive, can be:
:type day: datetime.date
:param day: Date to retrieve data for
:returns: list of tuples of (station, channel) as available.
.. note:: Currently the seishub options are untested.
"""
available_stations = []
if arc_type.lower() == 'day_vols':
wavefiles = glob.glob(os.path.join(archive, day.strftime('Y%Y'),
day.strftime('R%j.01'), '*'))
for wavefile in wavefiles:
header = read(wavefile, headonly=True)
available_stations.append((header[0].stats.station,
header[0].stats.channel))
elif arc_type.lower() == 'seishub':
client = SeishubClient(archive)
st = client.get_previews(starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + 86400)
for tr in st:
available_stations.append((tr.stats.station, tr.stats.channel))
elif arc_type.lower() == 'fdsn':
client = FDSNClient(archive)
inventory = client.get_stations(starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + 86400,
level='channel')
for network in inventory:
for station in network:
for channel in station:
available_stations.append((station.code,
channel.code))
return available_stations
def setUp(self):
self.client = Client(TESTSERVER)
class ClientTestCase(unittest.TestCase):
"""
Test cases for the SeisHub client.
"""
def setUp(self):
self.client = Client(TESTSERVER)
# def test_getWaveformApplyFilter(self):
# t = UTCDateTime("2009-09-03 00:00:00")
# #1 - w/o apply_filter
# st = self.client.waveform.get_waveforms("BW", "RTPI", "", "EHZ",
# t, t + 20, apply_filter=False)
# self.assertEqual(len(st), 1)
# self.assertEqual(st[0].stats.network, '')
# self.assertEqual(st[0].stats.station, 'GP01')
# self.assertEqual(st[0].stats.location, '')
# self.assertEqual(st[0].stats.channel, 'SHZ')
# #2 - w/ apply_filter
# st = self.client.waveform.get_waveforms("BW", "RTPI", "", "EHZ",
# t, t + 20, apply_filter=True)
# self.assertEqual(len(st), 1)
# self.assertEqual(st[0].stats.network, 'BW')
# self.assertEqual(st[0].stats.station, 'RTPI')
# self.assertEqual(st[0].stats.location, '')
# self.assertEqual(st[0].stats.channel, 'EHZ')
def test_get_event_list(self):
c = self.client.event
# UTCDateTimes
events = c.get_list(min_datetime=UTCDateTime("2009-01-01T00:00:00"),
max_datetime=UTCDateTime("2009-01-10T00:00:00"))
self.assertEqual(len(events), 4)
# time strings with T as separator
events = c.get_list(min_datetime="2009-01-01T00:00:00",
max_datetime="2009-01-10T00:00:00")
self.assertEqual(len(events), 4)
# time strings with space as separator
events = c.get_list(min_datetime="2009-01-01 00:00:00",
max_datetime="2009-01-10 00:00:00")
self.assertEqual(len(events), 4)
def test_get_network_ids(self):
items = ['KT', 'BW', 'CZ', 'GR', 'NZ']
data = self.client.waveform.get_network_ids()
for item in items:
self.assertIn(item, data)
def test_ping(self):
# current server
time = self.client.ping()
self.assertTrue(isinstance(time, float))
def test_get_station_ids(self):
# 1 - some selected stations
stations = ['FUR', 'FURT', 'ROTZ', 'RTAK', 'MANZ', 'WET']
data = self.client.waveform.get_station_ids()
for station in stations:
self.assertIn(station, data)
# 2 - all stations of network BW
stations = ['FURT', 'ROTZ', 'RTAK', 'MANZ']
data = self.client.waveform.get_station_ids(network='BW')
for station in stations:
self.assertIn(station, data)
def test_get_location_ids(self):
# 1 - all locations
items = ['', '10']
data = self.client.waveform.get_location_ids()
for item in items:
self.assertIn(item, data)
# 2 - all locations for network BW
items = ['']
data = self.client.waveform.get_location_ids(network='BW')
for item in items:
self.assertIn(item, data)
# 3 - all locations for network BW and station MANZ
items = ['']
data = self.client.waveform.get_location_ids(network='BW',
station='MANZ')
for item in items:
self.assertIn(item, data)
def test_get_channel_ids(self):
# 1 - all channels
items = ['AEX', 'AEY', 'BAN', 'BAZ', 'BHE', 'BHN', 'BHZ', 'EHE', 'EHN',
'EHZ', 'HHE', 'HHN', 'HHZ', 'LHE', 'LHN', 'LHZ', 'SHE', 'SHN',
'SHZ']
data = self.client.waveform.get_channel_ids()
for item in items:
self.assertIn(item, data)
# 2 - all channels for network BW
items = ['AEX', 'AEY', 'BAN', 'BAZ', 'BHE', 'BHN', 'BHZ', 'EHE', 'EHN',
'EHZ', 'HHE', 'HHN', 'HHZ', 'SHE', 'SHN', 'SHZ']
data = self.client.waveform.get_channel_ids(network='BW')
for item in items:
self.assertIn(item, data)
# 3 - all channels for network BW and station MANZ
items = ['AEX', 'AEY', 'EHE', 'EHN', 'EHZ', 'SHE', 'SHN', 'SHZ']
data = self.client.waveform.get_channel_ids(network='BW',
station='MANZ')
for item in items:
self.assertIn(item, data)
# 4 - all channels for network BW, station MANZ and given location
items = ['AEX', 'AEY', 'EHE', 'EHN', 'EHZ', 'SHE', 'SHN', 'SHZ']
data = self.client.waveform.get_channel_ids(
network='BW', station='MANZ', location='')
for item in items:
self.assertIn(item, data)
def test_get_preview(self):
# multiple channels / MiniSEED
t1 = UTCDateTime('20080101')
t2 = UTCDateTime('20080201')
st = self.client.waveform.get_previews("BW", "M*", "", "EHZ", t1, t2)
self.assertEqual(len(st), 4)
self.assertEqual(st[0].stats.network, 'BW')
self.assertEqual(st[0].stats.channel, 'EHZ')
self.assertEqual(st[0].stats.delta, 30.0)
# single channel / GSE2
t1 = UTCDateTime('20090101')
t2 = UTCDateTime('20100101')
st = self.client.waveform.get_previews("BW", "RTLI", "", "EHN", t1, t2)
self.assertEqual(len(st), 1)
self.assertEqual(st[0].id, 'BW.RTLI..EHN')
self.assertEqual(st[0].stats.delta, 30.0)
self.assertEqual(len(st[0]), 205642)
self.assertEqual(st[0].stats.npts, 205642)
def test_get_preview_by_ids(self):
# multiple channels / MiniSEED
t1 = UTCDateTime('20080101')
t2 = UTCDateTime('20080201')
# via list
st = self.client.waveform.get_previews_by_ids(
['BW.MANZ..EHE', 'BW.ROTZ..EHE'], t1, t2)
st.sort()
self.assertEqual(len(st), 2)
self.assertEqual(st[0].id, 'BW.MANZ..EHE')
self.assertEqual(st[1].id, 'BW.ROTZ..EHE')
# via string
st = self.client.waveform.get_previews_by_ids(
'BW.MANZ..EHE,BW.ROTZ..EHE', t1, t2)
st.sort()
self.assertEqual(len(st), 2)
self.assertEqual(st[0].id, 'BW.MANZ..EHE')
self.assertEqual(st[1].id, 'BW.ROTZ..EHE')
def test_get_paz(self):
t = UTCDateTime('20090808')
c = self.client
# test the deprecated call too for one/two releases
data = c.station.get_paz('BW.MANZ..EHZ', t)
self.assertEqual(data['zeros'], [0j, 0j])
self.assertEqual(data['sensitivity'], 2516800000.0)
self.assertEqual(len(data['poles']), 5)
self.assertEqual(data['poles'][0], (-0.037004 + 0.037016j))
self.assertEqual(data['poles'][1], (-0.037004 - 0.037016j))
self.assertEqual(data['poles'][2], (-251.33 + 0j))
self.assertEqual(data['poles'][3],
(-131.03999999999999 - 467.29000000000002j))
self.assertEqual(data['poles'][4],
(-131.03999999999999 + 467.29000000000002j))
self.assertEqual(data['gain'], 60077000.0)
# test some not allowed wildcards
t = UTCDateTime('20120501')
self.assertRaises(ValueError, c.station.get_paz, "BW.RLAS..BJ*", t)
self.assertRaises(ValueError, c.station.get_paz, "BW.RLAS..*", t)
self.assertRaises(ValueError, c.station.get_paz, "BW.RLAS..BJ?", t)
self.assertRaises(ValueError, c.station.get_paz, "BW.R*..BJZ", t)
# test with a XSEED file with a referenced PAZ response info (see #364)
t = UTCDateTime("2012-05-10")
result = AttribDict(
{'gain': 1.0, 'poles': [0j],
'sensitivity': 6319100000000.0, 'digitizer_gain': 1000000.0,
'seismometer_gain': 6319100.0, 'zeros': [0j]})
data = c.station.get_paz("BW.RLAS..BJZ", t)
self.assertEqual(data, result)
def test_get_coordinates(self):
t = UTCDateTime("2010-05-03T23:59:30")
data = self.client.station.get_coordinates(network="BW", station="UH1",
datetime=t, location="")
result = {'elevation': 500.0, 'latitude': 48.081493000000002,
'longitude': 11.636093000000001}
self.assertEqual(data, result)
def test_get_waveform_with_metadata(self):
# metadata change during t1 -> t2 !
t1 = UTCDateTime("2010-05-03T23:59:30")
t2 = UTCDateTime("2010-05-04T00:00:30")
client = self.client
self.assertRaises(Exception, client.waveform.get_waveforms, "BW",
"UH1", "", "EH*", t1, t2, get_paz=True,
get_coordinates=True)
st = client.waveform.get_waveforms("BW", "UH1", "", "EH*", t1, t2,
get_paz=True, get_coordinates=True,
metadata_timecheck=False)
result = AttribDict({'zeros': [0j, 0j, 0j], 'sensitivity': 251650000.0,
'poles': [(-0.88 + 0.88j), (-0.88 - 0.88j),
(-0.22 + 0j)],
'gain': 1.0,
'seismometer_gain': 400.0,
'digitizer_gain': 629121.0})
self.assertEqual(st[0].stats.paz, result)
result = AttribDict({'latitude': 48.081493000000002,
'elevation': 500.0,
'longitude': 11.636093000000001})
self.assertEqual(st[0].stats.coordinates, result)
def test_localcache(self):
"""
Tests local 'caching' of XML seed resources and station list coordinate
information to avoid repeat requests to server.
Tests..
- returned information is stored with client instance in memory
- repeat requests do not get stored duplicated locally
- repeat requests do not issue a request to server anymore
(- right results for example with two different metadata sets at
different times)
"""
net = "BW"
sta = "RTSA"
netsta = ".".join([net, sta])
seed_id = ".".join([net, sta, "", "EHZ"])
t1 = UTCDateTime("2009-09-01")
t2 = UTCDateTime("2012-10-23")
coords1 = dict(elevation=1022.0, latitude=47.7673, longitude=12.842417)
coords2 = dict(elevation=1066.0, latitude=47.768345,
longitude=12.841651)
paz1 = {'digitizer_gain': 16000000.0,
'gain': 1.0,
'poles': [(-0.88 + 0.88j), (-0.88 - 0.88j), (-0.22 + 0j)],
'seismometer_gain': 400.0,
'sensitivity': 6400000000.0,
'zeros': [0j, 0j, 0j]}
paz2 = {'digitizer_gain': 1677850.0,
'gain': 1.0,
'poles': [(-4.444 + 4.444j), (-4.444 - 4.444j), (-1.083 + 0j)],
'seismometer_gain': 400.0,
'sensitivity': 671140000.0,
'zeros': [0j, 0j, 0j]}
c = self.client
# before any requests
self.assertEqual(len(c.xml_seeds), 0)
self.assertEqual(len(c.station_list), 0)
# after first t1 requests
ret = c.station.get_coordinates(net, sta, t1)
self.assertEqual(ret, coords1)
self.assertEqual(len(c.station_list), 1)
self.assertEqual(len(c.station_list[netsta]), 1)
ret = c.station.get_paz(seed_id, t1)
self.assertEqual(ret, paz1)
self.assertEqual(len(c.xml_seeds), 1)
self.assertEqual(len(c.xml_seeds[seed_id]), 1)
# after first t2 requests
ret = c.station.get_coordinates(net, sta, t2)
self.assertEqual(ret, coords2)
self.assertEqual(len(c.station_list), 1)
self.assertEqual(len(c.station_list[netsta]), 2)
ret = c.station.get_paz(seed_id, t2)
self.assertEqual(ret, paz2)
self.assertEqual(len(c.xml_seeds), 1)
self.assertEqual(len(c.xml_seeds[seed_id]), 2)
# get_list() is called if get_paz or get_coordinates ends up making a
# request to server so we just overwrite it and let it raise to check
# that no request is issued
c.station.get_list = raise_on_call
# after second t1 requests
ret = c.station.get_coordinates(net, sta, t1)
self.assertEqual(ret, coords1)
self.assertEqual(len(c.station_list), 1)
self.assertEqual(len(c.station_list[netsta]), 2)
ret = c.station.get_paz(seed_id, t1)
self.assertEqual(ret, paz1)
self.assertEqual(len(c.xml_seeds), 1)
self.assertEqual(len(c.xml_seeds[seed_id]), 2)
# after second t2 requests
ret = c.station.get_coordinates(net, sta, t2)
self.assertEqual(ret, coords2)
self.assertEqual(len(c.station_list), 1)
self.assertEqual(len(c.station_list[netsta]), 2)
ret = c.station.get_paz(seed_id, t2)
self.assertEqual(ret, paz2)
self.assertEqual(len(c.xml_seeds), 1)
self.assertEqual(len(c.xml_seeds[seed_id]), 2)
# new request that needs to connect to server, just to make sure the
# monkey patch for raising on requests really works
self.assertRaises(RequestException, c.station.get_coordinates,
"GR", "FUR", t2)
self.assertRaises(RequestException, c.station.get_paz,
"GR.FUR..HHZ", t2)
def read_data(archive, arc_type, day, stachans, length=86400):
"""
Function to read the appropriate data from an archive for a day.
:type archive: str
:param archive:
The archive source - if arc_type is seishub, this should be a url,
if the arc_type is FDSN then this can be either a url or a known obspy
client. If arc_type is day_vols, then this is the path to the top
directory.
:type arc_type: str
:param arc_type: The type of archive, can be: seishub, FDSN, day_volumes
:type day: datetime.date
:param day: Date to retrieve data for
:type stachans: list
:param stachans: List of tuples of Stations and channels to try and get,
will not fail if stations are not available, but will warn.
:type length: float
:param length: Data length to extract in seconds, defaults to 1 day.
:returns: Stream of data
:rtype: obspy.core.stream.Stream
.. note:: A note on arc_types, if arc_type is day_vols, then this will \
look for directories labelled in the IRIS DMC conventions of \
Yyyyy/Rjjj.01/... where yyyy is the year and jjj is the julian day. \
Data within these files directories should be stored as day-long, \
single-channel files. This is not implemented in the fasted way \
possible to allow for a more general situation. If you require more \
speed you will need to re-write this.
.. rubric:: Example
>>> from obspy import UTCDateTime
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('JCNB', 'SP1')]
>>> st = read_data('NCEDC', 'FDSN', t1, stachans)
>>> print(st)
1 Trace(s) in Stream:
BP.JCNB.40.SP1 | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.\
950000Z | 20.0 Hz, 1728000 samples
.. rubric:: Example, missing data
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('JCNB', 'SP1'), ('GCSZ', 'HHZ')]
>>> st = read_data('NCEDC', 'FDSN', t1, stachans)
>>> print(st)
1 Trace(s) in Stream:
BP.JCNB.40.SP1 | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.\
950000Z | 20.0 Hz, 1728000 samples
.. rubric:: Example, local day-volumes
>>> # Get the path to the test data
>>> import eqcorrscan
>>> TEST_PATH = os.path.dirname(eqcorrscan.__file__) + '/tests/test_data'
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('WHYM', 'SHZ'), ('EORO', 'SHZ')]
>>> st = read_data(TEST_PATH + '/day_vols', 'day_vols',
... t1, stachans)
>>> print(st)
2 Trace(s) in Stream:
AF.WHYM..SHZ | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.000000Z \
| 1.0 Hz, 86400 samples
AF.EORO..SHZ | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.000000Z \
| 1.0 Hz, 86400 samples
"""
st = []
available_stations = _check_available_data(archive, arc_type, day)
for station in stachans:
if len(station[1]) == 2:
# Cope with two char channel naming in seisan
station_map = (station[0], station[1][0] + '*' + station[1][1])
available_stations_map = [(sta[0], sta[1][0] + '*' + sta[1][-1])
for sta in available_stations]
else:
station_map = station
available_stations_map = available_stations
if station_map not in available_stations_map:
msg = ' '.join([station[0], station_map[1], 'is not available for',
day.strftime('%Y/%m/%d')])
warnings.warn(msg)
continue
if arc_type.lower() == 'seishub':
client = SeishubClient(archive)
st += client.get_waveforms(
network='*', station=station_map[0], location='*',
channel=station_map[1], starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + length)
elif arc_type.upper() == "FDSN":
client = FDSNClient(archive)
try:
st += client.get_waveforms(
network='*', station=station_map[0], location='*',
channel=station_map[1], starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + length)
except FDSNException:
warnings.warn('No data on server despite station being ' +
'available...')
continue
elif arc_type.lower() == 'day_vols':
wavfiles = _get_station_file(os.path.join(
archive, day.strftime('Y%Y' + os.sep + 'R%j.01')),
station_map[0], station_map[1])
for wavfile in wavfiles:
st += read(wavfile, starttime=day, endtime=day + length)
st = Stream(st)
return st
def from_client(catalog, client_id, lowcut, highcut, samp_rate, filt_order,
length, prepick, swin, debug=0, plot=False):
r"""Function to generate templates from a SeisHub database.Must be given \
an obspy.Catalog class and the SeisHub url as input. The function returns \
a list of obspy.Stream classes containting steams for each desired \
template.
:type catalog: obspy.Catalog
:param catalog: Catalog class containing desired template events
:type url: string
:param url: url of SeisHub database instance
:type lowcut: float
:param lowcut: Low cut (Hz), if set to None will look in template\
defaults file
:type highcut: float
:param lowcut: High cut (Hz), if set to None will look in template\
defaults file
:type samp_rate: float
:param samp_rate: New sampling rate in Hz, if set to None will look in\
template defaults file
:type filt_order: int
:param filt_order: Filter level, if set to None will look in\
template defaults file
:type length: float
:param length: Extract length in seconds, if None will look in template\
defaults file.
:type prepick: float
:param prepick: Pre-pick time in seconds
:type swin: str
:param swin: Either 'all', 'P' or 'S', to select which phases to output.
:type debug: int
:param debug: Level of debugging output, higher=more
:type plot: bool
:param plot: Plot templates or not.
:returns: obspy.Stream Newly cut template
"""
# This import section copes with namespace changes between obspy versions
import obspy
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.fdsn import Client
from obspy.clients.fdsn.header import FDSNException
else:
from obspy.fdsn import Client
from obspy.fdsn.header import FDSNException
from eqcorrscan.utils import pre_processing
from obspy import UTCDateTime
import warnings
client = Client(client_id)
temp_list = []
for event in catalog:
# Figure out which picks we have
day = event.origins[0].time
print("Fetching the following traces from " + client_id)
for pick in event.picks:
net = pick.waveform_id.network_code
sta = pick.waveform_id.station_code
chan = pick.waveform_id.channel_code
loc = pick.waveform_id.location_code
starttime = UTCDateTime(pick.time.date)
endtime = starttime + 86400
# Here we download a full day of data. We do this so that minor
# differences in processing during processing due to the effect
# of resampling do not impinge on our cross-correaltions.
if debug > 0:
print('start-time: ' + str(starttime))
print('end-time: ' + str(endtime))
print('pick-time: ' + str(pick.time))
print('pick phase: ' + pick.phase_hint)
print('.'.join([net, sta, loc, chan]))
if 'st' not in locals():
try:
st = client.get_waveforms(net, sta, loc, chan,
starttime, endtime)
except FDSNException:
warnings.warn('Found no data for this station')
else:
try:
st += client.get_waveforms(net, sta, loc, chan,
starttime, endtime)
except FDSNException:
warnings.warn('Found no data for this station')
if debug > 0:
st.plot()
print('Pre-processing data for event: '+str(event.resource_id))
st.merge(fill_value='interpolate')
st1 = pre_processing.dayproc(st, lowcut, highcut, filt_order,
samp_rate, starttime=starttime,
debug=debug, parallel=True)
if debug > 0:
st1.plot()
template = _template_gen(event.picks, st1, length, swin, prepick,
plot)
del st, st1
temp_list.append(template)
return temp_list
def read_data(archive, arc_type, day, stachans, length=86400):
"""
Function to read the appropriate data from your archive for your selected \
day.
:type archive: str
:param archive: The archive source - if arc_type is seishub, this should \
be a url, if the arc_type is FDSN then this can be either a url or a \
known obspy client. If arc_type is day_vols, then this is the path \
to the top directory.
:type arc_type: str
:param arc_type: The type of archive, can be: seishub, FDSN, day_volves
:type day: datetime.date
:param day: Date to retrieve data for
:type stachans: list
:param stachans: List of tuples of Stations and channels to try and get,
will not fail if stations are not available, but will warn.
:type length: float
:param length: Data length to extract in seconds, defaults to 1 day.
:returns: obspy.core.stream.Stream
.. note:: A note on arc_types, if arc_type is day_vols, then this will \
look for directories labelled in the IRIS DMC conventions of \
Yyyyy/Rjjj.01/... where yyyy is the year and jjj is the julian day. \
Data within these files directories should be stored as day-long, \
single-channel files. This is not implemented in the fasted way \
possible to allow for a more general situation. If you require more \
speed you will need to re-write this.
.. rubric:: Example
>>> from eqcorrscan.utils.archive_read import read_data
>>> from obspy import UTCDateTime
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('FOZ', 'HHZ'), ('JCZ', 'HHZ')]
>>> st = read_data('GEONET', 'FDSN', t1, stachans)
>>> print(st)
2 Trace(s) in Stream:
NZ.FOZ.10.HHZ | 2012-03-25T23:59:57.018393Z - 2012-03-27T00:00:00.688393Z | 100.0 Hz, 8640368 samples
NZ.JCZ.10.HHZ | 2012-03-25T23:59:57.348391Z - 2012-03-27T00:00:02.958391Z | 100.0 Hz, 8640562 samples
.. rubric:: Example, missing data
>>> from eqcorrscan.utils.archive_read import read_data
>>> from obspy import UTCDateTime
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('FOZ', 'HHZ'), ('GCSZ', 'HHZ')]
>>> st = read_data('GEONET', 'FDSN', t1, stachans)
>>> print(st)
1 Trace(s) in Stream:
NZ.FOZ.10.HHZ | 2012-03-25T23:59:57.018393Z - 2012-03-27T00:00:00.688393Z | 100.0 Hz, 8640368 samples
.. rubric:: Example, local day-volumes
>>> from eqcorrscan.utils.archive_read import read_data
>>> from obspy import UTCDateTime
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('WHYM', 'SHZ'), ('EORO', 'SHZ')]
>>> st = read_data('eqcorrscan/tests/test_data/day_vols', 'day_vols',
... t1, stachans)
>>> print(st)
2 Trace(s) in Stream:
AF.WHYM..SHZ | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.000000Z | 1.0 Hz, 86400 samples
AF.EORO..SHZ | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.000000Z | 1.0 Hz, 86400 samples
"""
import obspy
import os
from obspy.clients.fdsn.header import FDSNException
if arc_type.lower() == 'seishub':
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.seishub import Client
else:
from obspy.seishub import Client
else:
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.fdsn import Client
else:
from obspy.fdsn import Client
from obspy import read, UTCDateTime
import warnings
st = []
available_stations = _check_available_data(archive, arc_type, day)
for station in stachans:
if len(station[1]) == 2:
# Cope with two char channel naming in seisan
station_map = (station[0], station[1][0] + '*' + station[1][1])
available_stations_map = [(sta[0], sta[1][0] + '*' + sta[1][-1])
for sta in available_stations]
else:
station_map = station
available_stations_map = available_stations
if station_map not in available_stations_map:
msg = ' '.join([
station[0], station_map[1], 'is not available for',
day.strftime('%Y/%m/%d')
])
warnings.warn(msg)
continue
if arc_type.lower() in ['seishub', 'fdsn']:
client = Client(archive)
try:
st += client.get_waveforms(network='*',
station=station_map[0],
location='*',
channel=station_map[1],
starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + length)
except FDSNException:
warnings.warn('No data on server despite station being ' +
'available...')
continue
elif arc_type.lower() == 'day_vols':
wavfiles = _get_station_file(
os.path.join(archive, day.strftime('Y%Y' + os.sep + 'R%j.01')),
station_map[0], station_map[1])
for wavfile in wavfiles:
st += read(wavfile, starttime=day, endtime=day + length)
st = obspy.Stream(st)
return st
def from_client(catalog, client_id, lowcut, highcut, samp_rate, filt_order,
length, prepick, swin, debug=0, plot=False):
r"""Function to generate templates from a SeisHub database.Must be given \
an obspy.Catalog class and the SeisHub url as input. The function returns \
a list of obspy.Stream classes containting steams for each desired \
template.
:type catalog: obspy.Catalog
:param catalog: Catalog class containing desired template events
:type url: string
:param url: url of SeisHub database instance
:type lowcut: float
:param lowcut: Low cut (Hz), if set to None will look in template\
defaults file
:type highcut: float
:param lowcut: High cut (Hz), if set to None will look in template\
defaults file
:type samp_rate: float
:param samp_rate: New sampling rate in Hz, if set to None will look in\
template defaults file
:type filt_order: int
:param filt_order: Filter level, if set to None will look in\
template defaults file
:type length: float
:param length: Extract length in seconds, if None will look in template\
defaults file.
:type prepick: float
:param prepick: Pre-pick time in seconds
:type swin: str
:param swin: Either 'all', 'P' or 'S', to select which phases to output.
:type debug: int
:param debug: Level of debugging output, higher=more
:type plot: bool
:param plot: Plot templates or not.
:returns: obspy.Stream Newly cut template
"""
# This import section copes with namespace changes between obspy versions
import obspy
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.fdsn import Client
from obspy.clients.fdsn.header import FDSNException
else:
from obspy.fdsn import Client
from obspy.fdsn.header import FDSNException
from eqcorrscan.utils import pre_processing
from obspy import UTCDateTime
import warnings
client = Client(client_id)
temp_list = []
for event in catalog:
# Figure out which picks we have
day = event.origins[0].time
print("Fetching the following traces from " + client_id)
for pick in event.picks:
net = pick.waveform_id.network_code
sta = pick.waveform_id.station_code
chan = pick.waveform_id.channel_code
loc = pick.waveform_id.location_code
starttime = UTCDateTime(pick.time.date)
endtime = starttime + 86400
# Here we download a full day of data. We do this so that minor
# differences in processing during processing due to the effect
# of resampling do not impinge on our cross-correaltions.
if debug > 0:
print('start-time: ' + str(starttime))
print('end-time: ' + str(endtime))
print('pick-time: ' + str(pick.time))
print('pick phase: ' + pick.phase_hint)
print('.'.join([net, sta, loc, chan]))
if 'st' not in locals():
try:
st = client.get_waveforms(net, sta, loc, chan,
starttime, endtime)
except FDSNException:
warnings.warn('Found no data for this station')
else:
try:
st += client.get_waveforms(net, sta, loc, chan,
starttime, endtime)
except FDSNException:
warnings.warn('Found no data for this station')
if debug > 0:
st.plot()
print('Pre-processing data for event: '+str(event.resource_id))
st.merge(fill_value='interpolate')
st1 = pre_processing.dayproc(st, lowcut, highcut, filt_order,
samp_rate, starttime=starttime,
debug=debug, parallel=True)
if debug > 0:
st1.plot()
template = _template_gen(event.picks, st1, length, swin, prepick,
plot=plot, debug=debug)
del st, st1
temp_list.append(template)
return temp_list
def read_data(archive, arc_type, day, stachans):
"""
Function to read the appropriate data from your archive for your selected \
day.
:type archive: str
:param archive: The archive source - if arc_type is seishub, this should \
be a url, if the arc_type is FDSN then this can be either a url or a \
known obspy client.
:type arc_type: str
:param arc_type: The type of archive, can be: seishub, FDSN, day_vols
:type day: datetime.date
:param day: Date to retrieve data for
:type stachans: list
:param stachans: List of tuples of Stations and channels to try and get,
will not fail if stations are not available, but will warn.
:returns: obspy.core.stream.Stream
.. note:: A note on arc_types, if arc_type is day_vols, then this will \
look for directories labelled in the IRIS DMC conventions of \
Yyyyy/Rjjj.01/... where yyyy is the year and jjj is the julian day. \
Data within these files directories should be stored as day-long, \
single-channel files. This is not implemented in the fasted way \
possible to allow for a more general situation. If you require more \
speed you will need to re-write this.
.. rubric:: Example
>>> from eqcorrscan.utils.archive_read import read_data
>>> from obspy import UTCDateTime
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('FOZ', 'HHZ'), ('JCZ', 'HHZ')]
>>> st = read_data('GEONET', 'FDSN', t1, stachans)
>>> print(st)
2 Trace(s) in Stream:
NZ.FOZ.10.HHZ | 2012-03-25T23:59:57.018393Z - 2012-03-27T00:00:00.688393Z | 100.0 Hz, 8640368 samples
NZ.JCZ.10.HHZ | 2012-03-25T23:59:57.348391Z - 2012-03-27T00:00:02.958391Z | 100.0 Hz, 8640562 samples
.. rubric:: Example, missing data
>>> from eqcorrscan.utils.archive_read import read_data
>>> from obspy import UTCDateTime
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('FOZ', 'HHZ'), ('GCSZ', 'HHZ')]
>>> st = read_data('GEONET', 'FDSN', t1, stachans)
>>> print(st)
1 Trace(s) in Stream:
NZ.FOZ.10.HHZ | 2012-03-25T23:59:57.018393Z - 2012-03-27T00:00:00.688393Z | 100.0 Hz, 8640368 samples
"""
import obspy
from obspy.clients.fdsn.header import FDSNException
if arc_type.lower() == 'seishub':
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.seishub import Client
else:
from obspy.seishub import Client
else:
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.fdsn import Client
else:
from obspy.fdsn import Client
from obspy import read, UTCDateTime
import warnings
st = []
available_stations = _check_available_data(archive, arc_type, day)
for station in stachans:
if len(station[1]) == 2:
# Cope with two char channel naming in seisan
station_map = (station[0], station[1][0] + '*' + station[1][1])
available_stations_map = [(sta[0], sta[1][0] + '*' + sta[1][-1])
for sta in available_stations]
else:
station_map = station
available_stations_map = available_stations
if station_map not in available_stations_map:
msg = ' '.join([station[0], station_map[1], 'is not available for',
day.strftime('%d/%m/%Y')])
warnings.warn(msg)
continue
if arc_type.lower() in ['seishub', 'fdsn']:
client = Client(archive)
try:
st += client.get_waveforms(network='*', station=station_map[0],
location='*',
channel=station_map[1],
starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + 86400)
except FDSNException:
warnings.warn('No data on server despite station being ' +
'available...')
continue
elif arc_type.lower() == 'day_vols':
wavfiles = _get_station_file(os.path.join(archive,
day.strftime('Y%Y' +
os.sep +
'R%j.01')),
station_map[0], station_map[1])
for wavfile in wavfiles:
st += read(wavfile)
st = obspy.Stream(st)
return st
def from_seishub(catalog,
url,
lowcut,
highcut,
samp_rate,
filt_order,
length,
prepick,
swin,
debug=0,
plot=False):
r"""Function to generate templates from a SeisHub database.Must be given \
an obspy.Catalog class and the SeisHub url as input. The function returns \
a list of obspy.Stream classes containting steams for each desired \
template.
:type catalog: obspy.Catalog
:param catalog: Catalog class containing desired template events
:type url: string
:param url: url of SeisHub database instance
:type lowcut: float
:param lowcut: Low cut (Hz), if set to None will look in template \
defaults file
:type highcut: float
:param lowcut: High cut (Hz), if set to None will look in template \
defaults file
:type samp_rate: float
:param samp_rate: New sampling rate in Hz, if set to None will look in \
template defaults file
:type filt_order: int
:param filt_order: Filter level, if set to None will look in \
template defaults file
:type length: float
:param length: Extract length in seconds, if None will look in template \
defaults file.
:type prepick: float
:param prepick: Pre-pick time in seconds
:type swin: str
:param swin: Either 'all', 'P' or 'S', to select which phases to output.
:type debug: int
:param debug: Level of debugging output, higher=more
:type plot: bool
:param plot: Plot templates or not.
:returns: obspy.Stream Newly cut template
"""
# This import section copes with namespace changes between obspy versions
import obspy
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.seishub import Client
else:
from obspy.seishub import Client
from eqcorrscan.utils import pre_processing
client = Client(url)
temp_list = []
for event in catalog:
# Figure out which picks we have
day = event.origins[0].time
picks = event.picks
print("Fetching the following traces from SeisHub")
for pick in picks:
net = pick.waveform_id.network_code
sta = pick.waveform_id.station_code
chan = pick.waveform_id.channel_code
loc = pick.waveform_id.location_code
starttime = pick.time - (prepick + 600)
# Enforce some pad, 10min either side, to reduce filter effects
endtime = pick.time + length + 600 - prepick
if debug > 0:
print('start-time: ' + str(starttime))
print('end-time: ' + str(endtime))
print('pick-time: ' + str(pick.time))
print('.'.join([net, sta, loc, chan]))
if sta in client.waveform.getStationIds(network=net):
if 'st' not in locals():
st = client.waveform.getWaveform(net, sta, loc, chan,
starttime, endtime)
else:
st += client.waveform.getWaveform(net, sta, loc, chan,
starttime, endtime)
else:
print('Station not found in SeisHub DB')
if debug > 0:
st.plot()
print('Preprocessing data for event: ' + str(event.resource_id))
st.merge(fill_value='interpolate')
st1 = pre_processing.shortproc(st, lowcut, highcut, filt_order,
samp_rate, debug)
template = _template_gen(event.picks,
st1,
length,
swin,
prepick,
plot=plot)
del st, st1
temp_list.append(template)
return temp_list
def from_client(catalog, client_id, lowcut, highcut, samp_rate, filt_order,
length, prepick, swin, debug=0, plot=False):
"""
Generate multiplexed template from FDSN client.
Function to generate templates from an FDSN client. Must be given \
an obspy.Catalog class and the client_id as input. The function returns \
a list of obspy.Stream classes containing steams for each desired \
template.
:type catalog: obspy.core.event.Catalog
:param catalog: Catalog class containing desired template events
:type client_id: str
:param client_id: Name of the client, either url, or Obspy \
mappable.
:type lowcut: float
:param lowcut: Low cut (Hz), if set to None will look in template\
defaults file
:type highcut: float
:param lowcut: High cut (Hz), if set to None will look in template\
defaults file
:type samp_rate: float
:param samp_rate: New sampling rate in Hz, if set to None will look in\
template defaults file
:type filt_order: int
:param filt_order: Filter level, if set to None will look in\
template defaults file
:type length: float
:param length: Extract length in seconds, if None will look in template\
defaults file.
:type prepick: float
:param prepick: Pre-pick time in seconds
:type swin: str
:param swin: Either 'all', 'P' or 'S', to select which phases to output.
:type debug: int
:param debug: Level of debugging output, higher=more
:type plot: bool
:param plot: Plot templates or not.
:returns: obspy.core.stream.Stream Newly cut template
.. rubric:: Example
>>> import obspy
>>> if int(obspy.__version__.split('.')[0]) >= 1:
... from obspy.clients.fdsn import Client
... else:
... from obspy.fdsn import Client
>>> from obspy.core.event import Catalog
>>> from eqcorrscan.core.template_gen import from_client
>>> client = Client('NCEDC')
>>> catalog = client.get_events(eventid='72572665', includearrivals=True)
>>> # We are only taking two picks for this example to speed up the example,
>>> # note that you don't have to!
>>> catalog[0].picks = catalog[0].picks[0:2]
>>> templates = from_client(catalog=catalog, client_id='NCEDC',
... lowcut=2.0, highcut=9.0, samp_rate=20.0,
... filt_order=4, length=3.0, prepick=0.15,
... swin='all')
Fetching the following traces from NCEDC
BG.CLV..DPZ
BK.BKS.00.HHZ
Pre-processing data for event: quakeml:nc.anss.org/Event/NC/72572665
>>> templates[0].plot(equal_scale=False, size=(800,600)) # doctest: +SKIP
.. figure:: ../../plots/template_gen.from_client.png
"""
# This import section copes with namespace changes between obspy versions
import obspy
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.fdsn import Client
from obspy.clients.fdsn.header import FDSNException
else:
from obspy.fdsn import Client
from obspy.fdsn.header import FDSNException
from eqcorrscan.utils import pre_processing
from obspy import UTCDateTime
import warnings
client = Client(client_id)
temp_list = []
for event in catalog:
# Figure out which picks we have
day = event.origins[0].time
print("Fetching the following traces from " + client_id)
dropped_pick_stations = 0
for pick in event.picks:
net = pick.waveform_id.network_code
sta = pick.waveform_id.station_code
chan = pick.waveform_id.channel_code
loc = pick.waveform_id.location_code
starttime = UTCDateTime(pick.time.date)
endtime = starttime + 86400
# Here we download a full day of data. We do this so that minor
# differences in processing during processing due to the effect
# of resampling do not impinge on our cross-correlations.
if debug > 0:
print('start-time: ' + str(starttime))
print('end-time: ' + str(endtime))
print('pick-time: ' + str(pick.time))
print('pick phase: ' + pick.phase_hint)
print('.'.join([net, sta, loc, chan]))
if 'st' not in locals():
try:
st = client.get_waveforms(net, sta, loc, chan,
starttime, endtime)
except FDSNException:
warnings.warn('Found no data for this station')
dropped_pick_stations += 1
else:
try:
st += client.get_waveforms(net, sta, loc, chan,
starttime, endtime)
except FDSNException:
warnings.warn('Found no data for this station')
dropped_pick_stations += 1
if debug > 0:
st.plot()
if not st and dropped_pick_stations == len(event.picks):
raise FDSNException('No data available, is the server down?')
print('Pre-processing data for event: '+str(event.resource_id))
st.merge(fill_value='interpolate')
st1 = pre_processing.dayproc(st, lowcut, highcut, filt_order,
samp_rate, starttime=starttime,
debug=debug, parallel=True)
if debug > 0:
st1.plot()
template = _template_gen(event.picks, st1, length, swin, prepick,
plot=plot, debug=debug)
del st, st1
temp_list.append(template)
return temp_list
def read_data(archive, arc_type, day, stachans):
"""
Function to read the appropriate data from your archive for your selected \
day.
:type archive: str
:param archive: The archive source - if arc_type is seishub, this should \
be a url, if the arc_type is FDSN then this can be either a url or a \
known obspy client.
:type arc_type: str
:param arc_type: The type of archive, can be: seishub, FDSN, day_vols
:type day: datetime.date
:param day: Date to retrieve data for
:type stations: list of tuple
:param station: Stations and channels to try and get, will not fail if \
stations are not available, but will warn.
:returns: obspy.Stream
.. note:: A note on arc_types, if arc_type is day_vols, then this will \
look for directories labelled in the IRIS DMC conventions of \
Yyyyy/Rjjj.01/... where yyyy is the year and jjj is the julian day. \
Data within these files directories should be stored as day-long, \
single-channel files. This is not implemented in the fasted way \
possible to allow for a more general situation. If you require more \
speed you will need to re-write this.
"""
import obspy
from obspy.clients.fdsn.header import FDSNException
if arc_type.lower() == 'seishub':
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.seishub import Client
else:
from obspy.seishub import Client
else:
if int(obspy.__version__.split('.')[0]) >= 1:
from obspy.clients.fdsn import Client
else:
from obspy.fdsn import Client
from obspy import read, UTCDateTime
import warnings
st = []
available_stations = _check_available_data(archive, arc_type, day)
for station in stachans:
if len(station[1]) == 2:
# Cope with two char channel naming in seisan
station_map = (station[0], station[1][0] + '*' + station[1][1])
available_stations_map = [(sta[0], sta[1][0] + '*' + sta[1][-1])
for sta in available_stations]
else:
station_map = station
available_stations_map = available_stations
if station_map not in available_stations_map:
msg = ' '.join([
station[0], station_map[1], 'is not available for',
day.strftime('%d/%m/%Y')
])
warnings.warn(msg)
continue
if arc_type.lower() in ['seishub', 'fdsn']:
client = Client(archive)
try:
st += client.get_waveforms(network='*',
station=station_map[0],
location='*',
channel=station_map[1],
starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + 86400)
except FDSNException:
warnings.warn('No data on server despite station being ' +
'available...')
continue
elif arc_type.lower() == 'day_vols':
wavfiles = _get_station_file(
os.path.join(archive, day.strftime('Y%Y' + os.sep + 'R%j.01')),
station_map[0], station_map[1])
for wavfile in wavfiles:
st += read(wavfile)
st = obspy.Stream(st)
return st