예제 #1
0
    def test_radial_queries(self):
        client = FDSNClient(self.live_server_url)
        lat = 48.995167 + 1.0
        lon = 11.519922

        self.assertEqual(
            len(
                client.get_stations(latitude=lat, longitude=lon,
                                    maxradius=2).get_contents()["stations"]),
            1)

        self.assertEqual(
            len(
                client.get_stations(latitude=lat, longitude=lon,
                                    maxradius=1.1).get_contents()["stations"]),
            1)

        with self.assertRaises(FDSNException):
            client.get_stations(latitude=lat,
                                longitude=lon,
                                minradius=1.1,
                                maxradius=10)

        with self.assertRaises(FDSNException):
            client.get_stations(latitude=lat,
                                longitude=lon,
                                minradius=0.1,
                                maxradius=0.5)
예제 #2
0
    def test_level_argument(self):
        client = FDSNClient(self.live_server_url)

        inv = client.get_stations(level="channel")
        c = inv.get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)

        inv = client.get_stations(level="station")
        c = inv.get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 0)

        inv = client.get_stations(level="network")
        c = inv.get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 0)
        self.assertEqual(len(c["channels"]), 0)

        inv = client.get_stations(level="response")
        c = inv.get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)

        for channel in inv[0][0]:
            self.assertIsNotNone(channel.response)
예제 #3
0
 def test_query_data(self):
     # query using ObsPy
     t1 = UTCDateTime("2005-10-06T07:21:59.850000")
     t2 = UTCDateTime("2005-10-06T07:24:59.845000")
     client = FDSNClient(self.live_server_url)
     got = client.get_waveforms("", "RJOB", "", "Z", t1, t2)[0]
     expected = read(FILES[0])[0]
     np.testing.assert_equal(got.data, expected.data)
     self.assertEqual(got, expected)
예제 #4
0
 def test_query_mapping(self):
     t1 = UTCDateTime(2010, 3, 25, 0, 0)
     t2 = t1 + 30
     client = FDSNClient(self.live_server_url)
     # 1 - direct query fails
     self.assertRaises(FDSNException,
                       client.get_waveforms, "TA", "*", "*", "BHE", t1, t2)
     # 2 - query use mapping works
     st = client.get_waveforms("XX", "YY", "00", "ZZZ", t1, t2)
     self.assertEqual(len(st), 1)
     # 3 - TA.A25A..BHZ and TA.A25A..BHN shouldn't be affected at all
     st = client.get_waveforms("TA", "A25A", "", "BH?", t1, t2)
     self.assertEqual(len(st), 2)
예제 #5
0
def _check_available_data(archive, arc_type, day):
    """
    Function to check what stations are available in the archive for a given \
    day.

    :type archive: str
    :param archive: The archive source
    :type arc_type: str
    :param arc_type: The type of archive, can be:
    :type day: datetime.date
    :param day: Date to retrieve data for

    :returns: list of tuples of (station, channel) as available.

    .. note:: Currently the seishub options are untested.

    """
    available_stations = []
    if arc_type.lower() == 'day_vols':
        wavefiles = glob.glob(
            os.path.join(archive, day.strftime('Y%Y'), day.strftime('R%j.01'),
                         '*'))
        for wavefile in wavefiles:
            header = read(wavefile, headonly=True)
            available_stations.append(
                (header[0].stats.station, header[0].stats.channel))
    elif arc_type.lower() == 'seishub':
        client = SeishubClient(archive)
        st = client.get_previews(starttime=UTCDateTime(day),
                                 endtime=UTCDateTime(day) + 86400)
        for tr in st:
            available_stations.append((tr.stats.station, tr.stats.channel))
    elif arc_type.lower() == 'fdsn':
        client = FDSNClient(archive)
        inventory = client.get_stations(starttime=UTCDateTime(day),
                                        endtime=UTCDateTime(day) + 86400,
                                        level='channel')
        for network in inventory:
            for station in network:
                for channel in station:
                    available_stations.append((station.code, channel.code))
    elif arc_type.upper() == "SDS":
        client = SDSClient(archive)
        nslc = client.get_all_nslc(sds_type=None, datetime=UTCDateTime(day))
        for item in nslc:
            available_stations.append((item[1], item[3]))
    return available_stations
예제 #6
0
    def test_no_wildcards(self):
        t1 = UTCDateTime(2010, 3, 25, 0, 0)
        t2 = t1 + 30
        client = FDSNClient(self.live_server_url)

        # network
        st = client.get_waveforms("TA", "*", "*", "*", t1, t2)
        self.assertEqual(len(st), 19)
        for id_ in ["T", "A", "X"]:
            self.assertRaises(FDSNException,
                              client.get_waveforms, id_, "*", "*", "*", t1, t2)
        # station
        st = client.get_waveforms("*", "A25A", "*", "*", t1, t2)
        self.assertEqual(len(st), 19)
        for id_ in ["A2", "25", "5A"]:
            self.assertRaises(FDSNException,
                              client.get_waveforms, "*", id_, "*", "*", t1, t2)
        # location
        st = client.get_waveforms("*", "*", "", "*", t1, t2)
        self.assertEqual(len(st), 19)
        # Jane does not distinguish between location ids with an empty
        # string or one or more spaces.
        # This is according to
        # http://www.fdsn.org/message-center/thread/108/
        st = client.get_waveforms("*", "*", "  ", "*", t1, t2)
        self.assertEqual(len(st), 19)
        st = client.get_waveforms("*", "*", " ", "*", t1, t2)
        self.assertEqual(len(st), 19)
        st = client.get_waveforms("*", "*", "--", "*", t1, t2)
        self.assertEqual(len(st), 19)
        for id_ in ["00", "XX", "X"]:
            self.assertRaises(FDSNException,
                              client.get_waveforms, "*", "*", id_, "*", t1, t2)
        # channel
        st = client.get_waveforms("*", "*", "*", "BHZ", t1, t2)
        self.assertEqual(len(st), 1)
        for id_ in ["B", "Z", "H"]:
            self.assertRaises(FDSNException,
                              client.get_waveforms, "*", "*", "*", id_, t1, t2)
예제 #7
0
 def test_query_data_wildcards(self):
     # query using wildcards
     t = UTCDateTime(2010, 3, 25, 0, 0)
     client = FDSNClient(self.live_server_url)
     # 1
     st = client.get_waveforms("TA", "A25A", "", "BHZ", t, t + 30)
     self.assertEqual(len(st), 1)
     self.assertEqual(len(st[0].data), 1201)
     self.assertEqual(st[0].id, 'TA.A25A..BHZ')
     # 2
     st = client.get_waveforms("TA", "A25A", "", "BHZ,BHN,BHE", t, t + 30)
     self.assertEqual(len(st), 3)
     # 3
     st = client.get_waveforms("TA", "A25A", "", "BH*", t, t + 30)
     self.assertEqual(len(st), 3)
     # 4
     st = client.get_waveforms("TA", "A25A", "", "BH?", t, t + 30)
     self.assertEqual(len(st), 3)
     # 5
     st = client.get_waveforms("TA", "A25A", "", "BH?,VCO", t, t + 30)
     self.assertEqual(len(st), 4)
     # 6
     st = client.get_waveforms("TA", "A25A", "", "BH?,-BHZ", t, t + 30)
     self.assertEqual(len(st), 2)
     # 7
     st = client.get_waveforms("TA", "A25A", "", "*", t, t + 30)
     self.assertEqual(len(st), 19)
     # 8
     st = client.get_waveforms("TA", "A25A", "", "*,-BHZ", t, t + 30)
     self.assertEqual(len(st), 18)
     # 9
     st = client.get_waveforms("*", "*", "*", "*", t, t + 30)
     self.assertEqual(len(st), 19)
     # 10
     st = client.get_waveforms("??", "????", "*", "???", t, t + 30)
     self.assertEqual(len(st), 19)
     # 11
     st = client.get_waveforms("?*", "?*?", "*", "?HZ", t, t + 30)
     self.assertEqual(len(st), 3)
예제 #8
0
    def test_rectangular_geo_queries(self):
        client = FDSNClient(self.live_server_url)
        # lat = 48.995167
        # lon = 11.519922

        # This works.
        self.assertEqual(
            len(
                client.get_stations(
                    minlatitude=48,
                    maxlatitude=49,
                    minlongitude=11,
                    maxlongitude=12).get_contents()["stations"]), 1)

        # Make sure one border does not include the point at a time.
        with self.assertRaises(FDSNException):
            client.get_stations(minlatitude=48.996,
                                maxlatitude=49,
                                minlongitude=11,
                                maxlongitude=12)

        with self.assertRaises(FDSNException):
            client.get_stations(minlatitude=48,
                                maxlatitude=48.5,
                                minlongitude=11,
                                maxlongitude=12)

        with self.assertRaises(FDSNException):
            client.get_stations(minlatitude=48,
                                maxlatitude=49,
                                minlongitude=11.6,
                                maxlongitude=12)

        with self.assertRaises(FDSNException):
            client.get_stations(minlatitude=48,
                                maxlatitude=49,
                                minlongitude=11,
                                maxlongitude=11.4)
예제 #9
0
    def test_total_and_selected_number_of_sta_and_cha(self):
        client = FDSNClient(self.live_server_url)

        inv = client.get_stations(level="network")
        self.assertEqual(inv[0].total_number_of_stations, 1)
        self.assertEqual(inv[0].selected_number_of_stations, 0)

        inv = client.get_stations(level="station")
        self.assertEqual(inv[0].total_number_of_stations, 1)
        self.assertEqual(inv[0].selected_number_of_stations, 1)
        self.assertEqual(inv[0][0].total_number_of_channels, 3)
        self.assertEqual(inv[0][0].selected_number_of_channels, 0)

        inv = client.get_stations(level="channel")
        self.assertEqual(inv[0].total_number_of_stations, 1)
        self.assertEqual(inv[0].selected_number_of_stations, 1)
        self.assertEqual(inv[0][0].total_number_of_channels, 3)
        self.assertEqual(inv[0][0].selected_number_of_channels, 3)

        inv = client.get_stations(level="response")
        self.assertEqual(inv[0].total_number_of_stations, 1)
        self.assertEqual(inv[0].selected_number_of_stations, 1)
        self.assertEqual(inv[0][0].total_number_of_channels, 3)
        self.assertEqual(inv[0][0].selected_number_of_channels, 3)
예제 #10
0
def read_data(archive, arc_type, day, stachans, length=86400):
    """
    Function to read the appropriate data from an archive for a day.

    :type archive: str
    :param archive:
        The archive source - if arc_type is seishub, this should be a url,
        if the arc_type is FDSN then this can be either a url or a known obspy
        client.  If arc_type is day_vols, then this is the path to the top
        directory.
    :type arc_type: str
    :param arc_type: The type of archive, can be: seishub, FDSN, day_volumes
    :type day: datetime.date
    :param day: Date to retrieve data for
    :type stachans: list
    :param stachans: List of tuples of Stations and channels to try and get,
        will not fail if stations are not available, but will warn.
    :type length: float
    :param length: Data length to extract in seconds, defaults to 1 day.

    :returns: Stream of data
    :rtype: obspy.core.stream.Stream

    .. note:: A note on arc_types, if arc_type is day_vols, then this will \
        look for directories labelled in the IRIS DMC conventions of \
        Yyyyy/Rjjj.01/... where yyyy is the year and jjj is the julian day. \
        Data within these files directories should be stored as day-long, \
        single-channel files.  This is not implemented in the fasted way \
        possible to allow for a more general situation.  If you require more \
        speed you will need to re-write this.

    .. rubric:: Example

    >>> from obspy import UTCDateTime
    >>> t1 = UTCDateTime(2012, 3, 26)
    >>> stachans = [('JCNB', 'SP1')]
    >>> st = read_data('NCEDC', 'FDSN', t1, stachans)
    >>> print(st)
    1 Trace(s) in Stream:
    BP.JCNB.40.SP1 | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.\
950000Z | 20.0 Hz, 1728000 samples

    .. rubric:: Example, missing data

    >>> t1 = UTCDateTime(2012, 3, 26)
    >>> stachans = [('JCNB', 'SP1'), ('GCSZ', 'HHZ')]
    >>> st = read_data('NCEDC', 'FDSN', t1, stachans)
    >>> print(st)
    1 Trace(s) in Stream:
    BP.JCNB.40.SP1 | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.\
950000Z | 20.0 Hz, 1728000 samples


    .. rubric:: Example, local day-volumes

    >>> # Get the path to the test data
    >>> import eqcorrscan
    >>> TEST_PATH = os.path.dirname(eqcorrscan.__file__) + '/tests/test_data'
    >>> t1 = UTCDateTime(2012, 3, 26)
    >>> stachans = [('WHYM', 'SHZ'), ('EORO', 'SHZ')]
    >>> st = read_data(TEST_PATH + '/day_vols', 'day_vols',
    ...                t1, stachans)
    >>> print(st)
    2 Trace(s) in Stream:
    AF.WHYM..SHZ | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.000000Z \
| 1.0 Hz, 86400 samples
    AF.EORO..SHZ | 2012-03-26T00:00:00.000000Z - 2012-03-26T23:59:59.000000Z \
| 1.0 Hz, 86400 samples
    """
    st = []
    available_stations = _check_available_data(archive, arc_type, day)
    for station in stachans:
        if len(station[1]) == 2:
            # Cope with two char channel naming in seisan
            station_map = (station[0], station[1][0] + '*' + station[1][1])
            available_stations_map = [(sta[0], sta[1][0] + '*' + sta[1][-1])
                                      for sta in available_stations]
        else:
            station_map = station
            available_stations_map = available_stations
        if station_map not in available_stations_map:
            msg = ' '.join([station[0], station_map[1], 'is not available for',
                            day.strftime('%Y/%m/%d')])
            warnings.warn(msg)
            continue
        if arc_type.lower() == 'seishub':
            client = SeishubClient(archive)
            st += client.get_waveforms(
                    network='*', station=station_map[0], location='*',
                    channel=station_map[1], starttime=UTCDateTime(day),
                    endtime=UTCDateTime(day) + length)
        elif arc_type.upper() == "FDSN":
            client = FDSNClient(archive)
            try:
                st += client.get_waveforms(
                    network='*', station=station_map[0], location='*',
                    channel=station_map[1], starttime=UTCDateTime(day),
                    endtime=UTCDateTime(day) + length)
            except FDSNException:
                warnings.warn('No data on server despite station being ' +
                              'available...')
                continue
        elif arc_type.lower() == 'day_vols':
            wavfiles = _get_station_file(os.path.join(
                archive, day.strftime('Y%Y' + os.sep + 'R%j.01')),
                station_map[0], station_map[1])
            for wavfile in wavfiles:
                st += read(wavfile, starttime=day, endtime=day + length)
    st = Stream(st)
    return st
예제 #11
0
def template_gen(method,
                 lowcut,
                 highcut,
                 samp_rate,
                 filt_order,
                 length,
                 prepick,
                 swin="all",
                 process_len=86400,
                 all_horiz=False,
                 delayed=True,
                 plot=False,
                 plotdir=None,
                 return_event=False,
                 min_snr=None,
                 parallel=False,
                 num_cores=False,
                 save_progress=False,
                 skip_short_chans=False,
                 **kwargs):
    """
    Generate processed and cut waveforms for use as templates.

    :type method: str
    :param method:
        Template generation method, must be one of ('from_client',
        'from_seishub', 'from_sac', 'from_meta_file'). - Each method requires
        associated arguments, see note below.
    :type lowcut: float
    :param lowcut: Low cut (Hz), if set to None will not apply a lowcut.
    :type highcut: float
    :param highcut: High cut (Hz), if set to None will not apply a highcut.
    :type samp_rate: float
    :param samp_rate: New sampling rate in Hz.
    :type filt_order: int
    :param filt_order: Filter level (number of corners).
    :type length: float
    :param length: Length of template waveform in seconds.
    :type prepick: float
    :param prepick: Pre-pick time in seconds
    :type swin: str
    :param swin:
        P, S, P_all, S_all or all, defaults to all: see note in
        :func:`eqcorrscan.core.template_gen.template_gen`
    :type process_len: int
    :param process_len: Length of data in seconds to download and process.
    :type all_horiz: bool
    :param all_horiz:
        To use both horizontal channels even if there is only a pick on one of
        them.  Defaults to False.
    :type delayed: bool
    :param delayed: If True, each channel will begin relative to it's own \
        pick-time, if set to False, each channel will begin at the same time.
    :type plot: bool
    :param plot: Plot templates or not.
    :type plotdir: str
	:param plotdir:
        The path to save plots to. If `plotdir=None` (default) then the figure
        will be shown on screen.
    :type return_event: bool
    :param return_event: Whether to return the event and process length or not.
    :type min_snr: float
    :param min_snr:
        Minimum signal-to-noise ratio for a channel to be included in the
        template, where signal-to-noise ratio is calculated as the ratio of
        the maximum amplitude in the template window to the rms amplitude in
        the whole window given.
    :type parallel: bool
    :param parallel: Whether to process data in parallel or not.
    :type num_cores: int
    :param num_cores:
        Number of cores to try and use, if False and parallel=True, will use
        either all your cores, or as many traces as in the data (whichever is
        smaller).
    :type save_progress: bool
    :param save_progress:
        Whether to save the resulting templates at every data step or not.
        Useful for long-running processes.
    :type skip_short_chans: bool
    :param skip_short_chans:
        Whether to ignore channels that have insufficient length data or not.
        Useful when the quality of data is not known, e.g. when downloading
        old, possibly triggered data from a datacentre

    :returns: List of :class:`obspy.core.stream.Stream` Templates
    :rtype: list

    .. note::
        *Method specific arguments:*

        - `from_client` requires:
            :param str client_id:
                string passable by obspy to generate Client, or a Client
                instance
            :param `obspy.core.event.Catalog` catalog:
                Catalog of events to generate template for
            :param float data_pad: Pad length for data-downloads in seconds
        - `from_seishub` requires:
            :param str url: url to seishub database
            :param `obspy.core.event.Catalog` catalog:
                Catalog of events to generate template for
            :param float data_pad: Pad length for data-downloads in seconds
        - `from_sac` requires:
            :param list sac_files:
                osbpy.core.stream.Stream of sac waveforms, or list of paths to
                sac waveforms.
            .. note::
                See `eqcorrscan.utils.sac_util.sactoevent` for details on
                how pick information is collected.
        - `from_meta_file` requires:
            :param str meta_file:
                Path to obspy-readable event file, or an obspy Catalog
            :param `obspy.core.stream.Stream` st:
                Stream containing waveform data for template. Note that this
                should be the same length of stream as you will use for the
                continuous detection, e.g. if you detect in day-long files,
                give this a day-long file!
            :param bool process:
                Whether to process the data or not, defaults to True.

    .. note::
        process_len should be set to the same length as used when computing
        detections using match_filter.match_filter, e.g. if you read
        in day-long data for match_filter, process_len should be 86400.

        .. rubric:: Example

    >>> from obspy.clients.fdsn import Client
    >>> from eqcorrscan.core.template_gen import template_gen
    >>> client = Client('NCEDC')
    >>> catalog = client.get_events(eventid='72572665', includearrivals=True)
    >>> # We are only taking two picks for this example to speed up the
    >>> # example, note that you don't have to!
    >>> catalog[0].picks = catalog[0].picks[0:2]
    >>> templates = template_gen(
    ...    method='from_client', catalog=catalog, client_id='NCEDC',
    ...    lowcut=2.0, highcut=9.0, samp_rate=20.0, filt_order=4, length=3.0,
    ...    prepick=0.15, swin='all', process_len=300, all_horiz=True)
    >>> templates[0].plot(equal_scale=False, size=(800,600)) # doctest: +SKIP

    .. figure:: ../../plots/template_gen.from_client.png

    .. rubric:: Example

    >>> from obspy import read
    >>> from eqcorrscan.core.template_gen import template_gen
    >>> # Get the path to the test data
    >>> import eqcorrscan
    >>> import os
    >>> TEST_PATH = os.path.dirname(eqcorrscan.__file__) + '/tests/test_data'
    >>> st = read(TEST_PATH + '/WAV/TEST_/' +
    ...           '2013-09-01-0410-35.DFDPC_024_00')
    >>> quakeml = TEST_PATH + '/20130901T041115.xml'
    >>> templates = template_gen(
    ...    method='from_meta_file', meta_file=quakeml, st=st, lowcut=2.0,
    ...    highcut=9.0, samp_rate=20.0, filt_order=3, length=2, prepick=0.1,
    ...    swin='S', all_horiz=True)
    >>> print(len(templates[0]))
    10
    >>> templates = template_gen(
    ...    method='from_meta_file', meta_file=quakeml, st=st, lowcut=2.0,
    ...    highcut=9.0, samp_rate=20.0, filt_order=3, length=2, prepick=0.1,
    ...    swin='S_all', all_horiz=True)
    >>> print(len(templates[0]))
    15

    .. rubric:: Example

    >>> from eqcorrscan.core.template_gen import template_gen
    >>> import glob
    >>> # Get all the SAC-files associated with one event.
    >>> sac_files = glob.glob(TEST_PATH + '/SAC/2014p611252/*')
    >>> templates = template_gen(
    ...    method='from_sac', sac_files=sac_files, lowcut=2.0, highcut=10.0,
    ...    samp_rate=25.0, filt_order=4, length=2.0, swin='all', prepick=0.1,
    ...    all_horiz=True)
    >>> print(templates[0][0].stats.sampling_rate)
    25.0
    >>> print(len(templates[0]))
    15
    """
    client_map = {'from_client': 'fdsn', 'from_seishub': 'seishub'}
    assert method in ('from_client', 'from_seishub', 'from_meta_file',
                      'from_sac')
    if not isinstance(swin, list):
        swin = [swin]
    process = True
    if method in ['from_client', 'from_seishub']:
        catalog = kwargs.get('catalog', Catalog())
        data_pad = kwargs.get('data_pad', 90)
        # Group catalog into days and only download the data once per day
        sub_catalogs = _group_events(catalog=catalog,
                                     process_len=process_len,
                                     template_length=length,
                                     data_pad=data_pad)
        if method == 'from_client':
            if isinstance(kwargs.get('client_id'), str):
                client = FDSNClient(kwargs.get('client_id', None))
            else:
                client = kwargs.get('client_id', None)
            available_stations = []
        else:
            client = SeisHubClient(kwargs.get('url', None), timeout=10)
            available_stations = client.waveform.get_station_ids()
    elif method == 'from_meta_file':
        if isinstance(kwargs.get('meta_file'), Catalog):
            catalog = kwargs.get('meta_file')
        elif kwargs.get('meta_file'):
            catalog = read_events(kwargs.get('meta_file'))
        elif kwargs.get('catalog'):
            catalog = kwargs.get('catalog')
        sub_catalogs = [catalog]
        st = kwargs.get('st', Stream())
        process = kwargs.get('process', True)
    elif method == 'from_sac':
        sac_files = kwargs.get('sac_files')
        if isinstance(sac_files, list):
            if isinstance(sac_files[0], (Stream, Trace)):
                # This is a list of streams...
                st = Stream(sac_files[0])
                for sac_file in sac_files[1:]:
                    st += sac_file
            else:
                sac_files = [read(sac_file)[0] for sac_file in sac_files]
                st = Stream(sac_files)
        else:
            st = sac_files
        # Make an event object...
        catalog = Catalog([sactoevent(st)])
        sub_catalogs = [catalog]

    temp_list = []
    process_lengths = []

    if "P_all" in swin or "S_all" in swin or all_horiz:
        all_channels = True
    else:
        all_channels = False
    for sub_catalog in sub_catalogs:
        if method in ['from_seishub', 'from_client']:
            Logger.info("Downloading data")
            st = _download_from_client(client=client,
                                       client_type=client_map[method],
                                       catalog=sub_catalog,
                                       data_pad=data_pad,
                                       process_len=process_len,
                                       available_stations=available_stations,
                                       all_channels=all_channels)
        Logger.info('Pre-processing data')
        st.merge()
        if len(st) == 0:
            Logger.info("No data")
            continue
        if process:
            data_len = max(
                [len(tr.data) / tr.stats.sampling_rate for tr in st])
            if 80000 < data_len < 90000:
                daylong = True
                starttime = min([tr.stats.starttime for tr in st])
                min_delta = min([tr.stats.delta for tr in st])
                # Cope with the common starttime less than 1 sample before the
                #  start of day.
                if (starttime + min_delta).date > starttime.date:
                    starttime = (starttime + min_delta)
                # Check if this is stupid:
                if abs(starttime - UTCDateTime(starttime.date)) > 600:
                    print(abs(starttime - UTCDateTime(starttime.date)))
                    daylong = False
                starttime = starttime.date
            else:
                daylong = False
            # Check if the required amount of data have been downloaded - skip
            # channels if arg set.
            if skip_short_chans:
                _st = Stream()
                for tr in st:
                    if np.ma.is_masked(tr.data):
                        _len = np.ma.count(tr.data) * tr.stats.delta
                    else:
                        _len = tr.stats.npts * tr.stats.delta
                    if _len < process_len * .8:
                        Logger.info(
                            "Data for {0} are too short, skipping".format(
                                tr.id))
                    else:
                        _st += tr
                st = _st
                if len(st) == 0:
                    Logger.info("No data")
                    continue
            if daylong:
                st = pre_processing.dayproc(st=st,
                                            lowcut=lowcut,
                                            highcut=highcut,
                                            filt_order=filt_order,
                                            samp_rate=samp_rate,
                                            parallel=parallel,
                                            starttime=UTCDateTime(starttime),
                                            num_cores=num_cores)
            else:
                st = pre_processing.shortproc(st=st,
                                              lowcut=lowcut,
                                              highcut=highcut,
                                              filt_order=filt_order,
                                              parallel=parallel,
                                              samp_rate=samp_rate,
                                              num_cores=num_cores)
        data_start = min([tr.stats.starttime for tr in st])
        data_end = max([tr.stats.endtime for tr in st])

        for event in sub_catalog:
            stations, channels, st_stachans = ([], [], [])
            if len(event.picks) == 0:
                Logger.warning('No picks for event {0}'.format(
                    event.resource_id))
                continue
            use_event = True
            # Check that the event is within the data
            for pick in event.picks:
                if not data_start < pick.time < data_end:
                    Logger.warning(
                        "Pick outside of data span: Pick time {0} Start "
                        "time {1} End time: {2}".format(
                            str(pick.time), str(data_start), str(data_end)))
                    use_event = False
            if not use_event:
                Logger.error('Event is not within data time-span')
                continue
            # Read in pick info
            Logger.debug("I have found the following picks")
            for pick in event.picks:
                if not pick.waveform_id:
                    Logger.warning(
                        'Pick not associated with waveforms, will not use:'
                        ' {0}'.format(pick))
                    continue
                Logger.debug(pick)
                stations.append(pick.waveform_id.station_code)
                channels.append(pick.waveform_id.channel_code)
            # Check to see if all picks have a corresponding waveform
            for tr in st:
                st_stachans.append('.'.join(
                    [tr.stats.station, tr.stats.channel]))
            # Cut and extract the templates
            template = _template_gen(event.picks,
                                     st,
                                     length,
                                     swin,
                                     prepick=prepick,
                                     plot=plot,
                                     all_horiz=all_horiz,
                                     delayed=delayed,
                                     min_snr=min_snr,
                                     plotdir=plotdir)
            process_lengths.append(len(st[0].data) / samp_rate)
            temp_list.append(template)
        if save_progress:
            if not os.path.isdir("eqcorrscan_temporary_templates"):
                os.makedirs("eqcorrscan_temporary_templates")
            for template in temp_list:
                template.write(
                    "eqcorrscan_temporary_templates{0}{1}.ms".format(
                        os.path.sep, template[0].stats.starttime),
                    format="MSEED")
        del st
    if return_event:
        return temp_list, catalog, process_lengths
    return temp_list
def custom_template_gen(method,
                        lowcut,
                        highcut,
                        samp_rate,
                        filt_order,
                        length,
                        prepick,
                        swin="all",
                        process_len=86400,
                        all_horiz=False,
                        delayed=True,
                        plot=False,
                        plotdir=None,
                        return_event=False,
                        min_snr=None,
                        parallel=False,
                        num_cores=False,
                        save_progress=False,
                        skip_short_chans=False,
                        **kwargs):
    """
    Generate processed and cut waveforms for use as templates.

    :type method: str
    :param method:
        Template generation method, must be one of ('from_client',
        'from_seishub', 'from_sac', 'from_meta_file'). - Each method requires
        associated arguments, see note below.
    :type lowcut: float
    :param lowcut: Low cut (Hz), if set to None will not apply a lowcut.
    :type highcut: float
    :param highcut: High cut (Hz), if set to None will not apply a highcut.
    :type samp_rate: float
    :param samp_rate: New sampling rate in Hz.
    :type filt_order: int
    :param filt_order: Filter level (number of corners).
    :type length: float
    :param length: Length of template waveform in seconds.
    :type prepick: float
    :param prepick: Pre-pick time in seconds
    :type swin: str
    :param swin:
        P, S, P_all, S_all or all, defaults to all: see note in
        :func:`eqcorrscan.core.template_gen.template_gen`
    :type process_len: int
    :param process_len: Length of data in seconds to download and process.
    :type all_horiz: bool
    :param all_horiz:
        To use both horizontal channels even if there is only a pick on one of
        them.  Defaults to False.
    :type delayed: bool
    :param delayed: If True, each channel will begin relative to it's own \
        pick-time, if set to False, each channel will begin at the same time.
    :type plot: bool
    :param plot: Plot templates or not.
    :type plotdir: str
    :param plotdir:
        The path to save plots to. If `plotdir=None` (default) then the figure
        will be shown on screen.
    :type return_event: bool
    :param return_event: Whether to return the event and process length or not.
    :type min_snr: float
    :param min_snr:
        Minimum signal-to-noise ratio for a channel to be included in the
        template, where signal-to-noise ratio is calculated as the ratio of
        the maximum amplitude in the template window to the rms amplitude in
        the whole window given.
    :type parallel: bool
    :param parallel: Whether to process data in parallel or not.
    :type num_cores: int
    :param num_cores:
        Number of cores to try and use, if False and parallel=True, will use
        either all your cores, or as many traces as in the data (whichever is
        smaller).
    :type save_progress: bool
    :param save_progress:
        Whether to save the resulting templates at every data step or not.
        Useful for long-running processes.
    :type skip_short_chans: bool
    :param skip_short_chans:
        Whether to ignore channels that have insufficient length data or not.
        Useful when the quality of data is not known, e.g. when downloading
        old, possibly triggered data from a datacentre

    :returns: List of :class:`obspy.core.stream.Stream` Templates
    :rtype: list

    """

    client_map = {'from_client': 'fdsn', 'from_seishub': 'seishub'}
    assert method in ('from_client', 'from_seishub', 'from_meta_file',
                      'from_sac')
    if not isinstance(swin, list):
        swin = [swin]
    process = True
    if method in ['from_client', 'from_seishub']:
        catalog = kwargs.get('catalog', Catalog())
        data_pad = kwargs.get('data_pad', 90)
        # Group catalog into days and only download the data once per day
        sub_catalogs = _group_events(catalog=catalog,
                                     process_len=process_len,
                                     template_length=length,
                                     data_pad=data_pad)
        if method == 'from_client':
            if isinstance(kwargs.get('client_id'), str):
                client = FDSNClient(kwargs.get('client_id', None))
            else:
                client = kwargs.get('client_id', None)
            available_stations = []
        else:
            client = SeisHubClient(kwargs.get('url', None), timeout=10)
            available_stations = client.waveform.get_station_ids()
    elif method == 'from_meta_file':
        if isinstance(kwargs.get('meta_file'), Catalog):
            catalog = kwargs.get('meta_file')
        elif kwargs.get('meta_file'):
            catalog = read_events(kwargs.get('meta_file'))
        else:
            catalog = kwargs.get('catalog')
        sub_catalogs = [catalog]
        st = kwargs.get('st', Stream())
        process = kwargs.get('process', True)
    elif method == 'from_sac':
        sac_files = kwargs.get('sac_files')
        if isinstance(sac_files, list):
            if isinstance(sac_files[0], (Stream, Trace)):
                # This is a list of streams...
                st = Stream(sac_files[0])
                for sac_file in sac_files[1:]:
                    st += sac_file
            else:
                sac_files = [read(sac_file)[0] for sac_file in sac_files]
                st = Stream(sac_files)
        else:
            st = sac_files
        # Make an event object...
        catalog = Catalog([sactoevent(st)])
        sub_catalogs = [catalog]

    temp_list = []
    process_lengths = []
    catalog_out = Catalog()

    if "P_all" in swin or "S_all" in swin or all_horiz:
        all_channels = True
    else:
        all_channels = False
    for sub_catalog in sub_catalogs:
        if method in ['from_seishub', 'from_client']:
            Logger.info("Downloading data")
            st = _download_from_client(client=client,
                                       client_type=client_map[method],
                                       catalog=sub_catalog,
                                       data_pad=data_pad,
                                       process_len=process_len,
                                       available_stations=available_stations,
                                       all_channels=all_channels)
        Logger.info('Pre-processing data')
        st.merge()
        if len(st) == 0:
            Logger.info("No data")
            continue
        if process:
            data_len = max(
                [len(tr.data) / tr.stats.sampling_rate for tr in st])
            if 80000 < data_len < 90000:
                daylong = True
                starttime = min([tr.stats.starttime for tr in st])
                min_delta = min([tr.stats.delta for tr in st])
                # Cope with the common starttime less than 1 sample before the
                #  start of day.
                if (starttime + min_delta).date > starttime.date:
                    starttime = (starttime + min_delta)
                # Check if this is stupid:
                if abs(starttime - UTCDateTime(starttime.date)) > 600:
                    daylong = False
                starttime = starttime.date
            else:
                daylong = False
            # Check if the required amount of data have been downloaded - skip
            # channels if arg set.
            for tr in st:
                if np.ma.is_masked(tr.data):
                    _len = np.ma.count(tr.data) * tr.stats.delta
                else:
                    _len = tr.stats.npts * tr.stats.delta
                if _len < process_len * .8:
                    Logger.info("Data for {0} are too short, skipping".format(
                        tr.id))
                    if skip_short_chans:
                        continue
                # Trim to enforce process-len
                tr.data = tr.data[0:int(process_len * tr.stats.sampling_rate)]
            if len(st) == 0:
                Logger.info("No data")
                continue
            if daylong:
                st = pre_processing.dayproc(st=st,
                                            lowcut=lowcut,
                                            highcut=highcut,
                                            filt_order=filt_order,
                                            samp_rate=samp_rate,
                                            parallel=parallel,
                                            starttime=UTCDateTime(starttime),
                                            num_cores=num_cores)
            else:
                st = pre_processing.shortproc(st=st,
                                              lowcut=lowcut,
                                              highcut=highcut,
                                              filt_order=filt_order,
                                              parallel=parallel,
                                              samp_rate=samp_rate,
                                              num_cores=num_cores)
        data_start = min([tr.stats.starttime for tr in st])
        data_end = max([tr.stats.endtime for tr in st])

        for event in sub_catalog:
            stations, channels, st_stachans = ([], [], [])
            if len(event.picks) == 0:
                Logger.warning('No picks for event {0}'.format(
                    event.resource_id))
                continue
            use_event = True
            # Check that the event is within the data
            for pick in event.picks:
                if not data_start < pick.time < data_end:
                    Logger.warning(
                        "Pick outside of data span: Pick time {0} Start "
                        "time {1} End time: {2}".format(
                            str(pick.time), str(data_start), str(data_end)))
                    use_event = False
            if not use_event:
                Logger.error('Event is not within data time-span')
                continue
            # Read in pick info
            Logger.debug("I have found the following picks")
            for pick in event.picks:
                if not pick.waveform_id:
                    Logger.warning(
                        'Pick not associated with waveforms, will not use:'
                        ' {0}'.format(pick))
                    continue
                Logger.debug(pick)
                stations.append(pick.waveform_id.station_code)
                channels.append(pick.waveform_id.channel_code)
            # Check to see if all picks have a corresponding waveform
            for tr in st:
                st_stachans.append('.'.join(
                    [tr.stats.station, tr.stats.channel]))
            # Cut and extract the templates
            template = _template_gen(event.picks,
                                     st,
                                     length,
                                     swin,
                                     prepick=prepick,
                                     plot=plot,
                                     all_horiz=all_horiz,
                                     delayed=delayed,
                                     min_snr=min_snr,
                                     plotdir=plotdir)
            process_lengths.append(len(st[0].data) / samp_rate)
            temp_list.append(template)
            catalog_out += event
        if save_progress:
            if not os.path.isdir("eqcorrscan_temporary_templates"):
                os.makedirs("eqcorrscan_temporary_templates")
            for template in temp_list:
                template.write(
                    "eqcorrscan_temporary_templates{0}{1}.ms".format(
                        os.path.sep, template[0].stats.starttime.strftime(
                            "%Y-%m-%dT%H%M%S")),
                    format="MSEED")
        del st
    if return_event:
        return temp_list, catalog_out, process_lengths
    return temp_list
예제 #13
0
    def test_seed_code_queries(self):
        client = FDSNClient(self.live_server_url)

        # First test some very specific queries.
        inv = client.get_stations(level="channel",
                                  network="BW",
                                  station="ALTM",
                                  location="--",
                                  channel="EH?")
        c = inv.get_contents()
        self.assertEqual(c["channels"],
                         ['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])

        client = FDSNClient(self.live_server_url)
        inv = client.get_stations(level="channel",
                                  network="BW",
                                  station="ALTM",
                                  location="--",
                                  channel="EH*")
        c = inv.get_contents()
        self.assertEqual(c["channels"],
                         ['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])

        client = FDSNClient(self.live_server_url)
        inv = client.get_stations(level="channel",
                                  network="BW",
                                  station="ALTM",
                                  location="",
                                  channel="EH*")
        c = inv.get_contents()
        self.assertEqual(c["channels"],
                         ['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])

        client = FDSNClient(self.live_server_url)
        inv = client.get_stations(level="channel",
                                  network="B*",
                                  station="AL?M",
                                  location="*",
                                  channel="EH*")
        c = inv.get_contents()
        self.assertEqual(c["channels"],
                         ['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])

        # Test exclusions. - First exclude things that don't exist in the
        # test database - should naturally still return everything.
        inv = client.get_stations(level="channel",
                                  network="-XX",
                                  station="-YY",
                                  location="-ZZ",
                                  channel="-BHE")
        c = inv.get_contents()
        self.assertEqual(c["channels"],
                         ['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])

        inv = client.get_stations(level="channel", channel="-EHE")
        c = inv.get_contents()
        self.assertEqual(c["channels"], ['BW.ALTM..EHN', 'BW.ALTM..EHZ'])

        inv = client.get_stations(level="channel", channel="-EHE,-EHN")
        c = inv.get_contents()
        self.assertEqual(c["channels"], ['BW.ALTM..EHZ'])

        # A couple of no-datas
        with self.assertRaises(FDSNException):
            client.get_stations(network="TA",
                                station="ALTM",
                                location="--",
                                channel="EH?")

        with self.assertRaises(FDSNException):
            client.get_stations(network="BW",
                                station="FURT",
                                location="--",
                                channel="EH?")

        with self.assertRaises(FDSNException):
            client.get_stations(network="BW",
                                station="ALTM",
                                location="00",
                                channel="EH?")

        with self.assertRaises(FDSNException):
            client.get_stations(network="BW",
                                station="ALTM",
                                location="--",
                                channel="BHZ?")
예제 #14
0
    def test_temporal_queries(self):
        """
        Test the various temporal parameters.
        """
        # All 3 channels start at the same time, two are open ended,
        # one ends a bit earlier.
        client = FDSNClient(self.live_server_url)

        start = obspy.UTCDateTime("2010-04-29T00:00:00.000000Z")
        end = obspy.UTCDateTime("2011-01-01T00:00:00.000000Z")

        inv = client.get_stations(starttime=obspy.UTCDateTime(2000, 1, 1),
                                  level="channel")
        c = inv.get_contents()
        self.assertEqual(c["channels"],
                         ['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)

        # Same thing.
        c = client.get_stations(starttime=start - 10,
                                level="channel").get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)

        # Go before and after the endtime of the one channel.
        c = client.get_stations(starttime=end - 10,
                                level="channel").get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)
        c = client.get_stations(starttime=end + 10,
                                level="channel").get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 2)

        # Test the endtime parameter.
        inv = client.get_stations(endtime=obspy.UTCDateTime(2016, 1, 1),
                                  level="channel")
        c = inv.get_contents()
        self.assertEqual(c["channels"],
                         ['BW.ALTM..EHE', 'BW.ALTM..EHN', 'BW.ALTM..EHZ'])
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)
        c = client.get_stations(endtime=start + 10,
                                level="channel").get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)
        with self.assertRaises(FDSNException):
            client.get_stations(endtime=start - 10, level="channel")

        # startbefore
        c = client.get_stations(startbefore=start + 10,
                                level="channel").get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)
        with self.assertRaises(FDSNException):
            client.get_stations(startbefore=start - 10, level="channel")

        # startafter
        c = client.get_stations(startafter=start - 10,
                                level="channel").get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)
        with self.assertRaises(FDSNException):
            client.get_stations(startafter=start + 10, level="channel")

        # endbefore
        c = client.get_stations(endbefore=end + 10,
                                level="channel").get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 1)
        with self.assertRaises(FDSNException):
            client.get_stations(endbefore=end - 10, level="channel")

        # endafter
        c = client.get_stations(endafter=end - 10,
                                level="channel").get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 3)
        c = client.get_stations(endafter=end + 10,
                                level="channel").get_contents()
        self.assertEqual(len(c["networks"]), 1)
        self.assertEqual(len(c["stations"]), 1)
        self.assertEqual(len(c["channels"]), 2)