def test_radial_queries(self):
client = FDSNClient(self.live_server_url)
lat = 48.995167 + 1.0
lon = 11.519922
self.assertEqual(
len(
client.get_stations(latitude=lat, longitude=lon,
maxradius=2).get_contents()["stations"]),
1)
self.assertEqual(
len(
client.get_stations(latitude=lat, longitude=lon,
maxradius=1.1).get_contents()["stations"]),
1)
with self.assertRaises(FDSNException):
client.get_stations(latitude=lat,
longitude=lon,
minradius=1.1,
maxradius=10)
with self.assertRaises(FDSNException):
client.get_stations(latitude=lat,
longitude=lon,
minradius=0.1,
maxradius=0.5)
def test_level_argument(self):
client = FDSNClient(self.live_server_url)
inv = client.get_stations(level="channel")
c = inv.get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
inv = client.get_stations(level="station")
c = inv.get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 0)
inv = client.get_stations(level="network")
c = inv.get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 0)
self.assertEqual(len(c["channels"]), 0)
inv = client.get_stations(level="response")
c = inv.get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
for channel in inv[0][0]:
self.assertIsNotNone(channel.response)
def test_query_data(self):
# query using ObsPy
t1 = UTCDateTime("2005-10-06T07:21:59.850000")
t2 = UTCDateTime("2005-10-06T07:24:59.845000")
client = FDSNClient(self.live_server_url)
got = client.get_waveforms("", "RJOB", "", "Z", t1, t2)[0]
expected = read(FILES[0])[0]
np.testing.assert_equal(got.data, expected.data)
self.assertEqual(got, expected)
def test_query_mapping(self):
t1 = UTCDateTime(2010, 3, 25, 0, 0)
t2 = t1 + 30
client = FDSNClient(self.live_server_url)
# 1 - direct query fails
self.assertRaises(FDSNException,
client.get_waveforms, "TA", "*", "*", "BHE", t1, t2)
# 2 - query use mapping works
st = client.get_waveforms("XX", "YY", "00", "ZZZ", t1, t2)
self.assertEqual(len(st), 1)
# 3 - TA.A25A..BHZ and TA.A25A..BHN shouldn't be affected at all
st = client.get_waveforms("TA", "A25A", "", "BH?", t1, t2)
self.assertEqual(len(st), 2)
def _check_available_data(archive, arc_type, day):
"""
Function to check what stations are available in the archive for a given \
day.
:type archive: str
:param archive: The archive source
:type arc_type: str
:param arc_type: The type of archive, can be:
:type day: datetime.date
:param day: Date to retrieve data for
:returns: list of tuples of (station, channel) as available.
.. note:: Currently the seishub options are untested.
"""
available_stations = []
if arc_type.lower() == 'day_vols':
wavefiles = glob.glob(
os.path.join(archive, day.strftime('Y%Y'), day.strftime('R%j.01'),
'*'))
for wavefile in wavefiles:
header = read(wavefile, headonly=True)
available_stations.append(
(header[0].stats.station, header[0].stats.channel))
elif arc_type.lower() == 'seishub':
client = SeishubClient(archive)
st = client.get_previews(starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + 86400)
for tr in st:
available_stations.append((tr.stats.station, tr.stats.channel))
elif arc_type.lower() == 'fdsn':
client = FDSNClient(archive)
inventory = client.get_stations(starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + 86400,
level='channel')
for network in inventory:
for station in network:
for channel in station:
available_stations.append((station.code, channel.code))
elif arc_type.upper() == "SDS":
client = SDSClient(archive)
nslc = client.get_all_nslc(sds_type=None, datetime=UTCDateTime(day))
for item in nslc:
available_stations.append((item[1], item[3]))
return available_stations
def test_no_wildcards(self):
t1 = UTCDateTime(2010, 3, 25, 0, 0)
t2 = t1 + 30
client = FDSNClient(self.live_server_url)
# network
st = client.get_waveforms("TA", "*", "*", "*", t1, t2)
self.assertEqual(len(st), 19)
for id_ in ["T", "A", "X"]:
self.assertRaises(FDSNException,
client.get_waveforms, id_, "*", "*", "*", t1, t2)
# station
st = client.get_waveforms("*", "A25A", "*", "*", t1, t2)
self.assertEqual(len(st), 19)
for id_ in ["A2", "25", "5A"]:
self.assertRaises(FDSNException,
client.get_waveforms, "*", id_, "*", "*", t1, t2)
# location
st = client.get_waveforms("*", "*", "", "*", t1, t2)
self.assertEqual(len(st), 19)
# Jane does not distinguish between location ids with an empty
# string or one or more spaces.
# This is according to
# http://www.fdsn.org/message-center/thread/108/
st = client.get_waveforms("*", "*", " ", "*", t1, t2)
self.assertEqual(len(st), 19)
st = client.get_waveforms("*", "*", " ", "*", t1, t2)
self.assertEqual(len(st), 19)
st = client.get_waveforms("*", "*", "--", "*", t1, t2)
self.assertEqual(len(st), 19)
for id_ in ["00", "XX", "X"]:
self.assertRaises(FDSNException,
client.get_waveforms, "*", "*", id_, "*", t1, t2)
# channel
st = client.get_waveforms("*", "*", "*", "BHZ", t1, t2)
self.assertEqual(len(st), 1)
for id_ in ["B", "Z", "H"]:
self.assertRaises(FDSNException,
client.get_waveforms, "*", "*", "*", id_, t1, t2)
def test_query_data_wildcards(self):
# query using wildcards
t = UTCDateTime(2010, 3, 25, 0, 0)
client = FDSNClient(self.live_server_url)
# 1
st = client.get_waveforms("TA", "A25A", "", "BHZ", t, t + 30)
self.assertEqual(len(st), 1)
self.assertEqual(len(st[0].data), 1201)
self.assertEqual(st[0].id, 'TA.A25A..BHZ')
# 2
st = client.get_waveforms("TA", "A25A", "", "BHZ,BHN,BHE", t, t + 30)
self.assertEqual(len(st), 3)
# 3
st = client.get_waveforms("TA", "A25A", "", "BH*", t, t + 30)
self.assertEqual(len(st), 3)
# 4
st = client.get_waveforms("TA", "A25A", "", "BH?", t, t + 30)
self.assertEqual(len(st), 3)
# 5
st = client.get_waveforms("TA", "A25A", "", "BH?,VCO", t, t + 30)
self.assertEqual(len(st), 4)
# 6
st = client.get_waveforms("TA", "A25A", "", "BH?,-BHZ", t, t + 30)
self.assertEqual(len(st), 2)
# 7
st = client.get_waveforms("TA", "A25A", "", "*", t, t + 30)
self.assertEqual(len(st), 19)
# 8
st = client.get_waveforms("TA", "A25A", "", "*,-BHZ", t, t + 30)
self.assertEqual(len(st), 18)
# 9
st = client.get_waveforms("*", "*", "*", "*", t, t + 30)
self.assertEqual(len(st), 19)
# 10
st = client.get_waveforms("??", "????", "*", "???", t, t + 30)
self.assertEqual(len(st), 19)
# 11
st = client.get_waveforms("?*", "?*?", "*", "?HZ", t, t + 30)
self.assertEqual(len(st), 3)
def test_rectangular_geo_queries(self):
client = FDSNClient(self.live_server_url)
# lat = 48.995167
# lon = 11.519922
# This works.
self.assertEqual(
len(
client.get_stations(
minlatitude=48,
maxlatitude=49,
minlongitude=11,
maxlongitude=12).get_contents()["stations"]), 1)
# Make sure one border does not include the point at a time.
with self.assertRaises(FDSNException):
client.get_stations(minlatitude=48.996,
maxlatitude=49,
minlongitude=11,
maxlongitude=12)
with self.assertRaises(FDSNException):
client.get_stations(minlatitude=48,
maxlatitude=48.5,
minlongitude=11,
maxlongitude=12)
with self.assertRaises(FDSNException):
client.get_stations(minlatitude=48,
maxlatitude=49,
minlongitude=11.6,
maxlongitude=12)
with self.assertRaises(FDSNException):
client.get_stations(minlatitude=48,
maxlatitude=49,
minlongitude=11,
maxlongitude=11.4)
def test_total_and_selected_number_of_sta_and_cha(self):
client = FDSNClient(self.live_server_url)
inv = client.get_stations(level="network")
self.assertEqual(inv[0].total_number_of_stations, 1)
self.assertEqual(inv[0].selected_number_of_stations, 0)
inv = client.get_stations(level="station")
self.assertEqual(inv[0].total_number_of_stations, 1)
self.assertEqual(inv[0].selected_number_of_stations, 1)
self.assertEqual(inv[0][0].total_number_of_channels, 3)
self.assertEqual(inv[0][0].selected_number_of_channels, 0)
inv = client.get_stations(level="channel")
self.assertEqual(inv[0].total_number_of_stations, 1)
self.assertEqual(inv[0].selected_number_of_stations, 1)
self.assertEqual(inv[0][0].total_number_of_channels, 3)
self.assertEqual(inv[0][0].selected_number_of_channels, 3)
inv = client.get_stations(level="response")
self.assertEqual(inv[0].total_number_of_stations, 1)
self.assertEqual(inv[0].selected_number_of_stations, 1)
self.assertEqual(inv[0][0].total_number_of_channels, 3)
self.assertEqual(inv[0][0].selected_number_of_channels, 3)
def read_data(archive, arc_type, day, stachans, length=86400):
"""
Function to read the appropriate data from an archive for a day.
:type archive: str
:param archive:
The archive source - if arc_type is seishub, this should be a url,
if the arc_type is FDSN then this can be either a url or a known obspy
client. If arc_type is day_vols, then this is the path to the top
directory.
:type arc_type: str
:param arc_type: The type of archive, can be: seishub, FDSN, day_volumes
:type day: datetime.date
:param day: Date to retrieve data for
:type stachans: list
:param stachans: List of tuples of Stations and channels to try and get,
will not fail if stations are not available, but will warn.
:type length: float
:param length: Data length to extract in seconds, defaults to 1 day.
:returns: Stream of data
:rtype: obspy.core.stream.Stream
.. note:: A note on arc_types, if arc_type is day_vols, then this will \
look for directories labelled in the IRIS DMC conventions of \
Yyyyy/Rjjj.01/... where yyyy is the year and jjj is the julian day. \
Data within these files directories should be stored as day-long, \
single-channel files. This is not implemented in the fasted way \
possible to allow for a more general situation. If you require more \
speed you will need to re-write this.
.. rubric:: Example
>>> from obspy import UTCDateTime
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('JCNB', 'SP1')]
>>> st = read_data('NCEDC', 'FDSN', t1, stachans)
>>> print(st)
1 Trace(s) in Stream:
BP.JCNB.40.SP1 | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.\
950000Z | 20.0 Hz, 1728000 samples
.. rubric:: Example, missing data
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('JCNB', 'SP1'), ('GCSZ', 'HHZ')]
>>> st = read_data('NCEDC', 'FDSN', t1, stachans)
>>> print(st)
1 Trace(s) in Stream:
BP.JCNB.40.SP1 | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.\
950000Z | 20.0 Hz, 1728000 samples
.. rubric:: Example, local day-volumes
>>> # Get the path to the test data
>>> import eqcorrscan
>>> TEST_PATH = os.path.dirname(eqcorrscan.__file__) + '/tests/test_data'
>>> t1 = UTCDateTime(2012, 3, 26)
>>> stachans = [('WHYM', 'SHZ'), ('EORO', 'SHZ')]
>>> st = read_data(TEST_PATH + '/day_vols', 'day_vols',
... t1, stachans)
>>> print(st)
2 Trace(s) in Stream:
AF.WHYM..SHZ | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.000000Z \
| 1.0 Hz, 86400 samples
AF.EORO..SHZ | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.000000Z \
| 1.0 Hz, 86400 samples
"""
st = []
available_stations = _check_available_data(archive, arc_type, day)
for station in stachans:
if len(station[1]) == 2:
# Cope with two char channel naming in seisan
station_map = (station[0], station[1][0] + '*' + station[1][1])
available_stations_map = [(sta[0], sta[1][0] + '*' + sta[1][-1])
for sta in available_stations]
else:
station_map = station
available_stations_map = available_stations
if station_map not in available_stations_map:
msg = ' '.join([station[0], station_map[1], 'is not available for',
day.strftime('%Y/%m/%d')])
warnings.warn(msg)
continue
if arc_type.lower() == 'seishub':
client = SeishubClient(archive)
st += client.get_waveforms(
network='*', station=station_map[0], location='*',
channel=station_map[1], starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + length)
elif arc_type.upper() == "FDSN":
client = FDSNClient(archive)
try:
st += client.get_waveforms(
network='*', station=station_map[0], location='*',
channel=station_map[1], starttime=UTCDateTime(day),
endtime=UTCDateTime(day) + length)
except FDSNException:
warnings.warn('No data on server despite station being ' +
'available...')
continue
elif arc_type.lower() == 'day_vols':
wavfiles = _get_station_file(os.path.join(
archive, day.strftime('Y%Y' + os.sep + 'R%j.01')),
station_map[0], station_map[1])
for wavfile in wavfiles:
st += read(wavfile, starttime=day, endtime=day + length)
st = Stream(st)
return st
def template_gen(method,
lowcut,
highcut,
samp_rate,
filt_order,
length,
prepick,
swin="all",
process_len=86400,
all_horiz=False,
delayed=True,
plot=False,
plotdir=None,
return_event=False,
min_snr=None,
parallel=False,
num_cores=False,
save_progress=False,
skip_short_chans=False,
**kwargs):
"""
Generate processed and cut waveforms for use as templates.
:type method: str
:param method:
Template generation method, must be one of ('from_client',
'from_seishub', 'from_sac', 'from_meta_file'). - Each method requires
associated arguments, see note below.
:type lowcut: float
:param lowcut: Low cut (Hz), if set to None will not apply a lowcut.
:type highcut: float
:param highcut: High cut (Hz), if set to None will not apply a highcut.
:type samp_rate: float
:param samp_rate: New sampling rate in Hz.
:type filt_order: int
:param filt_order: Filter level (number of corners).
:type length: float
:param length: Length of template waveform in seconds.
:type prepick: float
:param prepick: Pre-pick time in seconds
:type swin: str
:param swin:
P, S, P_all, S_all or all, defaults to all: see note in
:func:`eqcorrscan.core.template_gen.template_gen`
:type process_len: int
:param process_len: Length of data in seconds to download and process.
:type all_horiz: bool
:param all_horiz:
To use both horizontal channels even if there is only a pick on one of
them. Defaults to False.
:type delayed: bool
:param delayed: If True, each channel will begin relative to it's own \
pick-time, if set to False, each channel will begin at the same time.
:type plot: bool
:param plot: Plot templates or not.
:type plotdir: str
 :param plotdir:
The path to save plots to. If `plotdir=None` (default) then the figure
will be shown on screen.
:type return_event: bool
:param return_event: Whether to return the event and process length or not.
:type min_snr: float
:param min_snr:
Minimum signal-to-noise ratio for a channel to be included in the
template, where signal-to-noise ratio is calculated as the ratio of
the maximum amplitude in the template window to the rms amplitude in
the whole window given.
:type parallel: bool
:param parallel: Whether to process data in parallel or not.
:type num_cores: int
:param num_cores:
Number of cores to try and use, if False and parallel=True, will use
either all your cores, or as many traces as in the data (whichever is
smaller).
:type save_progress: bool
:param save_progress:
Whether to save the resulting templates at every data step or not.
Useful for long-running processes.
:type skip_short_chans: bool
:param skip_short_chans:
Whether to ignore channels that have insufficient length data or not.
Useful when the quality of data is not known, e.g. when downloading
old, possibly triggered data from a datacentre
:returns: List of :class:`obspy.core.stream.Stream` Templates
:rtype: list
.. note::
*Method specific arguments:*
- `from_client` requires:
:param str client_id:
string passable by obspy to generate Client, or a Client
instance
:param `obspy.core.event.Catalog` catalog:
Catalog of events to generate template for
:param float data_pad: Pad length for data-downloads in seconds
- `from_seishub` requires:
:param str url: url to seishub database
:param `obspy.core.event.Catalog` catalog:
Catalog of events to generate template for
:param float data_pad: Pad length for data-downloads in seconds
- `from_sac` requires:
:param list sac_files:
osbpy.core.stream.Stream of sac waveforms, or list of paths to
sac waveforms.
.. note::
See `eqcorrscan.utils.sac_util.sactoevent` for details on
how pick information is collected.
- `from_meta_file` requires:
:param str meta_file:
Path to obspy-readable event file, or an obspy Catalog
:param `obspy.core.stream.Stream` st:
Stream containing waveform data for template. Note that this
should be the same length of stream as you will use for the
continuous detection, e.g. if you detect in day-long files,
give this a day-long file!
:param bool process:
Whether to process the data or not, defaults to True.
.. note::
process_len should be set to the same length as used when computing
detections using match_filter.match_filter, e.g. if you read
in day-long data for match_filter, process_len should be 86400.
.. rubric:: Example
>>> from obspy.clients.fdsn import Client
>>> from eqcorrscan.core.template_gen import template_gen
>>> client = Client('NCEDC')
>>> catalog = client.get_events(eventid='72572665', includearrivals=True)
>>> # We are only taking two picks for this example to speed up the
>>> # example, note that you don't have to!
>>> catalog[0].picks = catalog[0].picks[0:2]
>>> templates = template_gen(
... method='from_client', catalog=catalog, client_id='NCEDC',
... lowcut=2.0, highcut=9.0, samp_rate=20.0, filt_order=4, length=3.0,
... prepick=0.15, swin='all', process_len=300, all_horiz=True)
>>> templates[0].plot(equal_scale=False, size=(800,600)) # doctest: +SKIP
.. figure:: ../../plots/template_gen.from_client.png
.. rubric:: Example
>>> from obspy import read
>>> from eqcorrscan.core.template_gen import template_gen
>>> # Get the path to the test data
>>> import eqcorrscan
>>> import os
>>> TEST_PATH = os.path.dirname(eqcorrscan.__file__) + '/tests/test_data'
>>> st = read(TEST_PATH + '/WAV/TEST_/' +
... '2013-09-01-0410-35.DFDPC_024_00')
>>> quakeml = TEST_PATH + '/20130901T041115.xml'
>>> templates = template_gen(
... method='from_meta_file', meta_file=quakeml, st=st, lowcut=2.0,
... highcut=9.0, samp_rate=20.0, filt_order=3, length=2, prepick=0.1,
... swin='S', all_horiz=True)
>>> print(len(templates[0]))
10
>>> templates = template_gen(
... method='from_meta_file', meta_file=quakeml, st=st, lowcut=2.0,
... highcut=9.0, samp_rate=20.0, filt_order=3, length=2, prepick=0.1,
... swin='S_all', all_horiz=True)
>>> print(len(templates[0]))
15
.. rubric:: Example
>>> from eqcorrscan.core.template_gen import template_gen
>>> import glob
>>> # Get all the SAC-files associated with one event.
>>> sac_files = glob.glob(TEST_PATH + '/SAC/2014p611252/*')
>>> templates = template_gen(
... method='from_sac', sac_files=sac_files, lowcut=2.0, highcut=10.0,
... samp_rate=25.0, filt_order=4, length=2.0, swin='all', prepick=0.1,
... all_horiz=True)
>>> print(templates[0][0].stats.sampling_rate)
25.0
>>> print(len(templates[0]))
15
"""
client_map = {'from_client': 'fdsn', 'from_seishub': 'seishub'}
assert method in ('from_client', 'from_seishub', 'from_meta_file',
'from_sac')
if not isinstance(swin, list):
swin = [swin]
process = True
if method in ['from_client', 'from_seishub']:
catalog = kwargs.get('catalog', Catalog())
data_pad = kwargs.get('data_pad', 90)
# Group catalog into days and only download the data once per day
sub_catalogs = _group_events(catalog=catalog,
process_len=process_len,
template_length=length,
data_pad=data_pad)
if method == 'from_client':
if isinstance(kwargs.get('client_id'), str):
client = FDSNClient(kwargs.get('client_id', None))
else:
client = kwargs.get('client_id', None)
available_stations = []
else:
client = SeisHubClient(kwargs.get('url', None), timeout=10)
available_stations = client.waveform.get_station_ids()
elif method == 'from_meta_file':
if isinstance(kwargs.get('meta_file'), Catalog):
catalog = kwargs.get('meta_file')
elif kwargs.get('meta_file'):
catalog = read_events(kwargs.get('meta_file'))
elif kwargs.get('catalog'):
catalog = kwargs.get('catalog')
sub_catalogs = [catalog]
st = kwargs.get('st', Stream())
process = kwargs.get('process', True)
elif method == 'from_sac':
sac_files = kwargs.get('sac_files')
if isinstance(sac_files, list):
if isinstance(sac_files[0], (Stream, Trace)):
# This is a list of streams...
st = Stream(sac_files[0])
for sac_file in sac_files[1:]:
st += sac_file
else:
sac_files = [read(sac_file)[0] for sac_file in sac_files]
st = Stream(sac_files)
else:
st = sac_files
# Make an event object...
catalog = Catalog([sactoevent(st)])
sub_catalogs = [catalog]
temp_list = []
process_lengths = []
if "P_all" in swin or "S_all" in swin or all_horiz:
all_channels = True
else:
all_channels = False
for sub_catalog in sub_catalogs:
if method in ['from_seishub', 'from_client']:
Logger.info("Downloading data")
st = _download_from_client(client=client,
client_type=client_map[method],
catalog=sub_catalog,
data_pad=data_pad,
process_len=process_len,
available_stations=available_stations,
all_channels=all_channels)
Logger.info('Pre-processing data')
st.merge()
if len(st) == 0:
Logger.info("No data")
continue
if process:
data_len = max(
[len(tr.data) / tr.stats.sampling_rate for tr in st])
if 80000 < data_len < 90000:
daylong = True
starttime = min([tr.stats.starttime for tr in st])
min_delta = min([tr.stats.delta for tr in st])
# Cope with the common starttime less than 1 sample before the
# start of day.
if (starttime + min_delta).date > starttime.date:
starttime = (starttime + min_delta)
# Check if this is stupid:
if abs(starttime - UTCDateTime(starttime.date)) > 600:
print(abs(starttime - UTCDateTime(starttime.date)))
daylong = False
starttime = starttime.date
else:
daylong = False
# Check if the required amount of data have been downloaded - skip
# channels if arg set.
if skip_short_chans:
_st = Stream()
for tr in st:
if np.ma.is_masked(tr.data):
_len = np.ma.count(tr.data) * tr.stats.delta
else:
_len = tr.stats.npts * tr.stats.delta
if _len < process_len * .8:
Logger.info(
"Data for {0} are too short, skipping".format(
tr.id))
else:
_st += tr
st = _st
if len(st) == 0:
Logger.info("No data")
continue
if daylong:
st = pre_processing.dayproc(st=st,
lowcut=lowcut,
highcut=highcut,
filt_order=filt_order,
samp_rate=samp_rate,
parallel=parallel,
starttime=UTCDateTime(starttime),
num_cores=num_cores)
else:
st = pre_processing.shortproc(st=st,
lowcut=lowcut,
highcut=highcut,
filt_order=filt_order,
parallel=parallel,
samp_rate=samp_rate,
num_cores=num_cores)
data_start = min([tr.stats.starttime for tr in st])
data_end = max([tr.stats.endtime for tr in st])
for event in sub_catalog:
stations, channels, st_stachans = ([], [], [])
if len(event.picks) == 0:
Logger.warning('No picks for event {0}'.format(
event.resource_id))
continue
use_event = True
# Check that the event is within the data
for pick in event.picks:
if not data_start < pick.time < data_end:
Logger.warning(
"Pick outside of data span: Pick time {0} Start "
"time {1} End time: {2}".format(
str(pick.time), str(data_start), str(data_end)))
use_event = False
if not use_event:
Logger.error('Event is not within data time-span')
continue
# Read in pick info
Logger.debug("I have found the following picks")
for pick in event.picks:
if not pick.waveform_id:
Logger.warning(
'Pick not associated with waveforms, will not use:'
' {0}'.format(pick))
continue
Logger.debug(pick)
stations.append(pick.waveform_id.station_code)
channels.append(pick.waveform_id.channel_code)
# Check to see if all picks have a corresponding waveform
for tr in st:
st_stachans.append('.'.join(
[tr.stats.station, tr.stats.channel]))
# Cut and extract the templates
template = _template_gen(event.picks,
st,
length,
swin,
prepick=prepick,
plot=plot,
all_horiz=all_horiz,
delayed=delayed,
min_snr=min_snr,
plotdir=plotdir)
process_lengths.append(len(st[0].data) / samp_rate)
temp_list.append(template)
if save_progress:
if not os.path.isdir("eqcorrscan_temporary_templates"):
os.makedirs("eqcorrscan_temporary_templates")
for template in temp_list:
template.write(
"eqcorrscan_temporary_templates{0}{1}.ms".format(
os.path.sep, template[0].stats.starttime),
format="MSEED")
del st
if return_event:
return temp_list, catalog, process_lengths
return temp_list
def custom_template_gen(method,
lowcut,
highcut,
samp_rate,
filt_order,
length,
prepick,
swin="all",
process_len=86400,
all_horiz=False,
delayed=True,
plot=False,
plotdir=None,
return_event=False,
min_snr=None,
parallel=False,
num_cores=False,
save_progress=False,
skip_short_chans=False,
**kwargs):
"""
Generate processed and cut waveforms for use as templates.
:type method: str
:param method:
Template generation method, must be one of ('from_client',
'from_seishub', 'from_sac', 'from_meta_file'). - Each method requires
associated arguments, see note below.
:type lowcut: float
:param lowcut: Low cut (Hz), if set to None will not apply a lowcut.
:type highcut: float
:param highcut: High cut (Hz), if set to None will not apply a highcut.
:type samp_rate: float
:param samp_rate: New sampling rate in Hz.
:type filt_order: int
:param filt_order: Filter level (number of corners).
:type length: float
:param length: Length of template waveform in seconds.
:type prepick: float
:param prepick: Pre-pick time in seconds
:type swin: str
:param swin:
P, S, P_all, S_all or all, defaults to all: see note in
:func:`eqcorrscan.core.template_gen.template_gen`
:type process_len: int
:param process_len: Length of data in seconds to download and process.
:type all_horiz: bool
:param all_horiz:
To use both horizontal channels even if there is only a pick on one of
them. Defaults to False.
:type delayed: bool
:param delayed: If True, each channel will begin relative to it's own \
pick-time, if set to False, each channel will begin at the same time.
:type plot: bool
:param plot: Plot templates or not.
:type plotdir: str
:param plotdir:
The path to save plots to. If `plotdir=None` (default) then the figure
will be shown on screen.
:type return_event: bool
:param return_event: Whether to return the event and process length or not.
:type min_snr: float
:param min_snr:
Minimum signal-to-noise ratio for a channel to be included in the
template, where signal-to-noise ratio is calculated as the ratio of
the maximum amplitude in the template window to the rms amplitude in
the whole window given.
:type parallel: bool
:param parallel: Whether to process data in parallel or not.
:type num_cores: int
:param num_cores:
Number of cores to try and use, if False and parallel=True, will use
either all your cores, or as many traces as in the data (whichever is
smaller).
:type save_progress: bool
:param save_progress:
Whether to save the resulting templates at every data step or not.
Useful for long-running processes.
:type skip_short_chans: bool
:param skip_short_chans:
Whether to ignore channels that have insufficient length data or not.
Useful when the quality of data is not known, e.g. when downloading
old, possibly triggered data from a datacentre
:returns: List of :class:`obspy.core.stream.Stream` Templates
:rtype: list
"""
client_map = {'from_client': 'fdsn', 'from_seishub': 'seishub'}
assert method in ('from_client', 'from_seishub', 'from_meta_file',
'from_sac')
if not isinstance(swin, list):
swin = [swin]
process = True
if method in ['from_client', 'from_seishub']:
catalog = kwargs.get('catalog', Catalog())
data_pad = kwargs.get('data_pad', 90)
# Group catalog into days and only download the data once per day
sub_catalogs = _group_events(catalog=catalog,
process_len=process_len,
template_length=length,
data_pad=data_pad)
if method == 'from_client':
if isinstance(kwargs.get('client_id'), str):
client = FDSNClient(kwargs.get('client_id', None))
else:
client = kwargs.get('client_id', None)
available_stations = []
else:
client = SeisHubClient(kwargs.get('url', None), timeout=10)
available_stations = client.waveform.get_station_ids()
elif method == 'from_meta_file':
if isinstance(kwargs.get('meta_file'), Catalog):
catalog = kwargs.get('meta_file')
elif kwargs.get('meta_file'):
catalog = read_events(kwargs.get('meta_file'))
else:
catalog = kwargs.get('catalog')
sub_catalogs = [catalog]
st = kwargs.get('st', Stream())
process = kwargs.get('process', True)
elif method == 'from_sac':
sac_files = kwargs.get('sac_files')
if isinstance(sac_files, list):
if isinstance(sac_files[0], (Stream, Trace)):
# This is a list of streams...
st = Stream(sac_files[0])
for sac_file in sac_files[1:]:
st += sac_file
else:
sac_files = [read(sac_file)[0] for sac_file in sac_files]
st = Stream(sac_files)
else:
st = sac_files
# Make an event object...
catalog = Catalog([sactoevent(st)])
sub_catalogs = [catalog]
temp_list = []
process_lengths = []
catalog_out = Catalog()
if "P_all" in swin or "S_all" in swin or all_horiz:
all_channels = True
else:
all_channels = False
for sub_catalog in sub_catalogs:
if method in ['from_seishub', 'from_client']:
Logger.info("Downloading data")
st = _download_from_client(client=client,
client_type=client_map[method],
catalog=sub_catalog,
data_pad=data_pad,
process_len=process_len,
available_stations=available_stations,
all_channels=all_channels)
Logger.info('Pre-processing data')
st.merge()
if len(st) == 0:
Logger.info("No data")
continue
if process:
data_len = max(
[len(tr.data) / tr.stats.sampling_rate for tr in st])
if 80000 < data_len < 90000:
daylong = True
starttime = min([tr.stats.starttime for tr in st])
min_delta = min([tr.stats.delta for tr in st])
# Cope with the common starttime less than 1 sample before the
# start of day.
if (starttime + min_delta).date > starttime.date:
starttime = (starttime + min_delta)
# Check if this is stupid:
if abs(starttime - UTCDateTime(starttime.date)) > 600:
daylong = False
starttime = starttime.date
else:
daylong = False
# Check if the required amount of data have been downloaded - skip
# channels if arg set.
for tr in st:
if np.ma.is_masked(tr.data):
_len = np.ma.count(tr.data) * tr.stats.delta
else:
_len = tr.stats.npts * tr.stats.delta
if _len < process_len * .8:
Logger.info("Data for {0} are too short, skipping".format(
tr.id))
if skip_short_chans:
continue
# Trim to enforce process-len
tr.data = tr.data[0:int(process_len * tr.stats.sampling_rate)]
if len(st) == 0:
Logger.info("No data")
continue
if daylong:
st = pre_processing.dayproc(st=st,
lowcut=lowcut,
highcut=highcut,
filt_order=filt_order,
samp_rate=samp_rate,
parallel=parallel,
starttime=UTCDateTime(starttime),
num_cores=num_cores)
else:
st = pre_processing.shortproc(st=st,
lowcut=lowcut,
highcut=highcut,
filt_order=filt_order,
parallel=parallel,
samp_rate=samp_rate,
num_cores=num_cores)
data_start = min([tr.stats.starttime for tr in st])
data_end = max([tr.stats.endtime for tr in st])
for event in sub_catalog:
stations, channels, st_stachans = ([], [], [])
if len(event.picks) == 0:
Logger.warning('No picks for event {0}'.format(
event.resource_id))
continue
use_event = True
# Check that the event is within the data
for pick in event.picks:
if not data_start < pick.time < data_end:
Logger.warning(
"Pick outside of data span: Pick time {0} Start "
"time {1} End time: {2}".format(
str(pick.time), str(data_start), str(data_end)))
use_event = False
if not use_event:
Logger.error('Event is not within data time-span')
continue
# Read in pick info
Logger.debug("I have found the following picks")
for pick in event.picks:
if not pick.waveform_id:
Logger.warning(
'Pick not associated with waveforms, will not use:'
' {0}'.format(pick))
continue
Logger.debug(pick)
stations.append(pick.waveform_id.station_code)
channels.append(pick.waveform_id.channel_code)
# Check to see if all picks have a corresponding waveform
for tr in st:
st_stachans.append('.'.join(
[tr.stats.station, tr.stats.channel]))
# Cut and extract the templates
template = _template_gen(event.picks,
st,
length,
swin,
prepick=prepick,
plot=plot,
all_horiz=all_horiz,
delayed=delayed,
min_snr=min_snr,
plotdir=plotdir)
process_lengths.append(len(st[0].data) / samp_rate)
temp_list.append(template)
catalog_out += event
if save_progress:
if not os.path.isdir("eqcorrscan_temporary_templates"):
os.makedirs("eqcorrscan_temporary_templates")
for template in temp_list:
template.write(
"eqcorrscan_temporary_templates{0}{1}.ms".format(
os.path.sep, template[0].stats.starttime.strftime(
"%Y-%m-%dT%H%M%S")),
format="MSEED")
del st
if return_event:
return temp_list, catalog_out, process_lengths
return temp_list
def test_seed_code_queries(self):
client = FDSNClient(self.live_server_url)
# First test some very specific queries.
inv = client.get_stations(level="channel",
network="BW",
station="ALTM",
location="--",
channel="EH?")
c = inv.get_contents()
self.assertEqual(c["channels"],
['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])
client = FDSNClient(self.live_server_url)
inv = client.get_stations(level="channel",
network="BW",
station="ALTM",
location="--",
channel="EH*")
c = inv.get_contents()
self.assertEqual(c["channels"],
['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])
client = FDSNClient(self.live_server_url)
inv = client.get_stations(level="channel",
network="BW",
station="ALTM",
location="",
channel="EH*")
c = inv.get_contents()
self.assertEqual(c["channels"],
['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])
client = FDSNClient(self.live_server_url)
inv = client.get_stations(level="channel",
network="B*",
station="AL?M",
location="*",
channel="EH*")
c = inv.get_contents()
self.assertEqual(c["channels"],
['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])
# Test exclusions. - First exclude things that don't exist in the
# test database - should naturally still return everything.
inv = client.get_stations(level="channel",
network="-XX",
station="-YY",
location="-ZZ",
channel="-BHE")
c = inv.get_contents()
self.assertEqual(c["channels"],
['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])
inv = client.get_stations(level="channel", channel="-EHE")
c = inv.get_contents()
self.assertEqual(c["channels"], ['BW.ALTM..EHN', 'BW.ALTM..EHZ'])
inv = client.get_stations(level="channel", channel="-EHE,-EHN")
c = inv.get_contents()
self.assertEqual(c["channels"], ['BW.ALTM..EHZ'])
# A couple of no-datas
with self.assertRaises(FDSNException):
client.get_stations(network="TA",
station="ALTM",
location="--",
channel="EH?")
with self.assertRaises(FDSNException):
client.get_stations(network="BW",
station="FURT",
location="--",
channel="EH?")
with self.assertRaises(FDSNException):
client.get_stations(network="BW",
station="ALTM",
location="00",
channel="EH?")
with self.assertRaises(FDSNException):
client.get_stations(network="BW",
station="ALTM",
location="--",
channel="BHZ?")
def test_temporal_queries(self):
"""
Test the various temporal parameters.
"""
# All 3 channels start at the same time, two are open ended,
# one ends a bit earlier.
client = FDSNClient(self.live_server_url)
start = obspy.UTCDateTime("2010-04-29T00:00:00.000000Z")
end = obspy.UTCDateTime("2011-01-01T00:00:00.000000Z")
inv = client.get_stations(starttime=obspy.UTCDateTime(2000, 1, 1),
level="channel")
c = inv.get_contents()
self.assertEqual(c["channels"],
['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
# Same thing.
c = client.get_stations(starttime=start - 10,
level="channel").get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
# Go before and after the endtime of the one channel.
c = client.get_stations(starttime=end - 10,
level="channel").get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
c = client.get_stations(starttime=end + 10,
level="channel").get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 2)
# Test the endtime parameter.
inv = client.get_stations(endtime=obspy.UTCDateTime(2016, 1, 1),
level="channel")
c = inv.get_contents()
self.assertEqual(c["channels"],
['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
c = client.get_stations(endtime=start + 10,
level="channel").get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
with self.assertRaises(FDSNException):
client.get_stations(endtime=start - 10, level="channel")
# startbefore
c = client.get_stations(startbefore=start + 10,
level="channel").get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
with self.assertRaises(FDSNException):
client.get_stations(startbefore=start - 10, level="channel")
# startafter
c = client.get_stations(startafter=start - 10,
level="channel").get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
with self.assertRaises(FDSNException):
client.get_stations(startafter=start + 10, level="channel")
# endbefore
c = client.get_stations(endbefore=end + 10,
level="channel").get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 1)
with self.assertRaises(FDSNException):
client.get_stations(endbefore=end - 10, level="channel")
# endafter
c = client.get_stations(endafter=end - 10,
level="channel").get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 3)
c = client.get_stations(endafter=end + 10,
level="channel").get_contents()
self.assertEqual(len(c["networks"]), 1)
self.assertEqual(len(c["stations"]), 1)
self.assertEqual(len(c["channels"]), 2)