예제 #1
0
 def test_warn_zeros(self):
     st = self.st.copy()
     template = _template_gen(self.picks, st.copy(), 10)
     self.assertTrue('LABE' in [tr.stats.station for tr in template])
     st.select(station='LABE', channel='SN')[0].data = np.zeros(10000)
     template = _template_gen(self.picks, st, 10)
     self.assertFalse('LABE' in [tr.stats.station for tr in template])
예제 #2
0
 def test_swin_P(self):
     p_picks = []
     for pick in self.picks:
         if pick.phase_hint == 'P':
             p_picks.append(pick)
     template = _template_gen(self.picks, self.st.copy(), 10, swin='P')
     self.assertEqual(len(template), len(p_picks))
     used_stations = [tr.stats.station for tr in template]
     for pick in p_picks:
         self.assertTrue(pick.waveform_id.station_code in used_stations)
예제 #3
0
 def test_warn_no_phase_hint(self):
     picks = copy.deepcopy(self.picks)
     for pick in picks:
         setattr(pick, 'phase_hint', None)
     with warnings.catch_warnings(record=True) as w:
         template = _template_gen(picks, self.st.copy(), 10)
     self.assertGreater(len(w), 0)
     self.assertEqual(len(template), 11)
     _w = ' '.join([warning.message.__str__() for warning in w])
     self.assertTrue("no phase hint given" in _w)
예제 #4
0
 def test_warn_no_phase_hint(self):
     picks = copy.deepcopy(self.picks)
     for pick in picks:
         setattr(pick, 'phase_hint', None)
     template = _template_gen(picks, self.st.copy(), 10)
     w = self.log_messages['warning']
     self.assertGreater(len(w), 0)
     self.assertEqual(len(template), 11)
     _w = ' '.join([warning for warning in w])
     self.assertTrue("no phase hint given" in _w)
예제 #5
0
 def test_swin_all_and_all_horiz(self):
     template = _template_gen(self.picks,
                              self.st.copy(),
                              10,
                              swin='all',
                              all_horiz=True)
     for pick in self.picks:
         if pick.phase_hint == 'S':
             self.assertGreaterEqual(
                 len(template.select(
                     station=pick.waveform_id.station_code)), 2)
예제 #6
0
 def test_extract_from_stack(self):
     length = 3
     stack = self.st.copy()
     template = _template_gen(self.picks, self.st.copy(), 2)
     extracted = extract_from_stack(stack,
                                    template,
                                    length=length,
                                    pre_pick=0.3,
                                    pre_pad=45)
     self.assertEqual(len(template), len(extracted))
     for tr in extracted:
         self.assertEqual(tr.stats.endtime - tr.stats.starttime, length)
예제 #7
0
 def setUpClass(cls):
     test_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                              'test_data', 'REA', 'TEST_',
                              '01-0411-15L.S201309')
     test_wavefile = os.path.join(
         os.path.dirname(os.path.abspath(__file__)), 'test_data', 'WAV',
         'TEST_', '2013-09-01-0410-35.DFDPC_024_00')
     cls.event = read_events(test_file)[0]
     cls.st = read(test_wavefile)
     cls.st.detrend().filter("bandpass", freqmin=2, freqmax=20, corners=4)
     for tr in cls.st:
         tr = tr.trim(tr.stats.starttime + 30, tr.stats.endtime - 30)
         # Hack around seisan 2-letter channel naming
         tr.stats.channel = tr.stats.channel[0] + tr.stats.channel[-1]
     cls.template = template_gen._template_gen(cls.event.picks, cls.st, 2)
예제 #8
0
 def test_extract_from_stack_and_process(self):
     length = 3
     stack = self.st.copy()
     template = _template_gen(self.picks, self.st.copy(), 2)
     extracted = extract_from_stack(stack,
                                    template,
                                    length=length,
                                    pre_pick=0.3,
                                    pre_pad=45,
                                    pre_processed=False,
                                    samp_rate=20,
                                    lowcut=2,
                                    highcut=8)
     self.assertEqual(len(template), len(extracted))
     for tr in extracted:
         self.assertEqual(tr.stats.endtime - tr.stats.starttime, length)
예제 #9
0
 def test_no_channel_code(self):
     picks = copy.deepcopy(self.picks)
     for pick in picks:
         setattr(pick.waveform_id, 'channel_code', None)
     template = _template_gen(picks, self.st.copy(), 10)
     self.assertEqual(len(template), 0)
예제 #10
0
def match_synth(sfile, cont_base, freqmin=2.0, freqmax=10.0, samp_rate=100.0,\
                threshold=8.0, threshold_type='MAD', trig_int=6.0, plotvar=True,\
                save_template=True):
    """
    Function to generate a basic synthetic from a real event, given by an s-file
    and cross-correlate this with the day of continuous data including the event

    :type sfile: str
    :param sfile: Path to the s-file for the event
    :type cont_base: str
    :param cont_base: Path to the continuous data, should be in Yyyyy/Rjjj.01\
                directories
    :type freqmin: float
    :param freqmin: Low-cut for bandpass in Hz, defualts to 2.0
    :type freqmax: float
    :param freqmax: High-cut for bandpass in Hz, defaults to 10.0
    :type samp_rate: float
    :param samp_rate: Desired sampling rate in Hz, defaults to 100.0
    :type threshold: float
    :param threshold: Threshold for detection in cccsum, defaults to 8.0
    :type threshold_type: str
    :param threshold_type: Type to threshold, either MAD or ABS, defaults to MAD
    :type trig_int: float
    :param trig_int: Trigger interval in seconds, defaults to 6.0
    :type plotvar: bool
    :param plotvar: To plot or not, defaults to true

    :returns: detections
    """
    # import matplotlib.pyplot as plt
    from eqcorrscan.core import match_filter, template_gen
    from eqcorrscan.utils import Sfile_util, pre_processing
    import glob
    from obspy import read, Stream, UTCDateTime
    from obspy.signal.cross_correlation import xcorr
    from joblib import Parallel, delayed
    from multiprocessing import cpu_count
    import numpy as np
    # Generate the synthetic
    synth_template=synth_from_sfile(sfile, samp_rate, length=1.0,\
                                    PS_ratio=1.68)
    synth_template.filter('bandpass', freqmin=freqmin, freqmax=freqmax)
    for tr in synth_template:
        tr.data=(tr.data*1000).astype(np.int32)
    # Find the date from the sfile
    event_date=Sfile_util.readheader(sfile).time.datetime
    day=UTCDateTime(event_date.date())
    # Work out which stations we have template info for
    stachans=[(tr.stats.station, tr.stats.channel) for tr in synth_template]
    # Read in the day of data
    for stachan in stachans:
        wavfile=glob.glob(cont_base+event_date.strftime('/Y%Y/R%j.01/')+\
                            stachan[0]+'.*.'+stachan[1][0]+'?'+stachan[1][-1]+\
                            '.'+event_date.strftime('%Y.%j'))
        if len(wavfile) != 0:
            for wavf in wavfile:
                if not 'st' in locals():
                    st=read(wavf)
                else:
                    st+=read(wavf)
    st=st.merge(fill_value='interpolate')
    cores=cpu_count()
    if len(st) < cores:
        jobs=len(st)
    else:
        jobs=cores
    st=Parallel(n_jobs=jobs)(delayed(pre_processing.dayproc)(tr, freqmin,\
                                                             freqmax, 3,\
                                                             samp_rate, 0,\
                                                             day)
                            for tr in st)
    st=Stream(st)
    # Make the real template
    picks=Sfile_util.readpicks(sfile)
    real_template=template_gen._template_gen(picks, st, 1.0, 'all',\
                                            prepick=10/samp_rate)
    for tr in real_template:
        tr.data=tr.data.astype(np.int32)
    if save_template:
        real_template.write('Real_'+sfile.split('/')[-1], format='MSEED',\
                            encoding='STEIM2')
    # Shift the synthetic to better align with the real one
    for tr in real_template:
        synth_tr=synth_template.select(station=tr.stats.station,\
                                        channel=tr.stats.channel)[0]
        shift, corr = xcorr(tr.data, synth_tr.data, 20)
        print tr.stats.station+'.'+tr.stats.channel+\
            ' shift='+str(shift)+'samples corr='+str(corr)
        if corr < 0:
            synth_tr.data*=-1
        # Apply a pad
        pad=np.zeros(abs(shift))
        if shift < 0:
            synth_tr.data=np.append(synth_tr.data, pad)[abs(shift):]
        elif shift > 0:
            synth_tr.data=np.append(pad, synth_tr.data)[0:-shift]
    if save_template:
        synth_template.write('Synthetic_'+sfile.split('/')[-1],
                            format='MSEED', encoding='STEIM2')
    # Now we have processed data and a template, we can try and detect!
    detections=match_filter.match_filter(['Synthetic_'+sfile.split('/')[-1],
                                        'Real_'+sfile.split('/')[-1]],\
                                        [synth_template, real_template],\
                                        st, threshold, \
                                        threshold_type, trig_int,\
                                        plotvar, 'synth_temp')
    f=open('Synthetic_test.csv', 'w')
    f.write('template, detect-time, cccsum, threshold, number of channels\n')
    for detection in detections:
        # output detections to file
        f.write(detection.template_name+', '+str(detection.detect_time)+\
                ', '+str(detection.detect_val)+', '+str(detection.threshold)+\
                ', '+str(detection.no_chans)+'\n')
        print 'template: '+detection.template_name+' detection at: '\
            +str(detection.detect_time)+' with a cccsum of: '+\
            str(detection.detect_val)
    if detections:
        f.write('\n')
    f.close()
예제 #11
0
def brightness(stations, nodes, lags, stream, threshold, thresh_type,
               template_length, template_saveloc, coherence_thresh,
               coherence_stations=['all'], coherence_clip=False,
               gap=2.0, clip_level=100, instance=0, pre_pick=0.2,
               plotsave=True, cores=1):
    r"""Function to calculate the brightness function in terms of energy for \
    a day of data over the entire network for a given grid of nodes.

    Note data in stream must be all of the same length and have the same
    sampling rates.

    :type stations: list
    :param stations: List of station names from in the form where stations[i] \
        refers to nodes[i][:] and lags[i][:]
    :type nodes: list, tuple
    :param nodes: List of node points where nodes[i] referes to stations[i] \
        and nodes[:][:][0] is latitude in degrees, nodes[:][:][1] is \
        longitude in degrees, nodes[:][:][2] is depth in km.
    :type lags: :class: 'numpy.array'
    :param lags: Array of arrays where lags[i][:] refers to stations[i]. \
        lags[i][j] should be the delay to the nodes[i][j] for stations[i] in \
        seconds.
    :type stream: :class: `obspy.Stream`
    :param data: Data through which to look for detections.
    :type threshold: float
    :param threshold: Threshold value for detection of template within the \
        brightness function
    :type thresh_type: str
    :param thresh_type: Either MAD or abs where MAD is the Median Absolute \
        Deviation and abs is an absoulte brightness.
    :type template_length: float
    :param template_length: Length of template to extract in seconds
    :type template_saveloc: str
    :param template_saveloc: Path of where to save the templates.
    :type coherence_thresh: tuple of floats
    :param coherence_thresh: Threshold for removing incoherant peaks in the \
            network response, those below this will not be used as templates. \
            Must be in the form of (a,b) where the coherence is given by: \
            a-kchan/b where kchan is the number of channels used to compute \
            the coherence
    :type coherence_stations: list
    :param coherence_stations: List of stations to use in the coherance \
            thresholding - defaults to 'all' which uses all the stations.
    :type coherence_clip: float
    :param coherence_clip: tuple
    :type coherence_clip: Start and end in seconds of data to window around, \
            defaults to False, which uses all the data given.
    :type pre_pick: float
    :param pre_pick: Seconds before the detection time to include in template
    :type plotsave: bool
    :param plotsave: Save or show plots, if False will try and show the plots \
            on screen - as this is designed for bulk use this is set to \
            True to save any plots rather than show them if you create \
            them - changes the backend of matplotlib, so if is set to \
            False you will see NO PLOTS!
    :type cores: int
    :param core: Number of cores to use, defaults to 1.
    :type clip_level: float
    :param clip_level: Multiplier applied to the mean deviation of the energy \
                    as an upper limit, used to remove spikes (earthquakes, \
                    lightning, electircal spikes) from the energy stack.
    :type gap: float
    :param gap: Minimum inter-event time in seconds for detections

    :return: list of templates as :class: `obspy.Stream` objects
    """
    from eqcorrscan.core.template_gen import _template_gen
    if plotsave:
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
        plt.ioff()
    # from joblib import Parallel, delayed
    from multiprocessing import Pool, cpu_count
    from copy import deepcopy
    from obspy import read as obsread
    from obspy.core.event import Catalog, Event, Pick, WaveformStreamID, Origin
    from obspy.core.event import EventDescription, CreationInfo, Comment
    import obspy.Stream
    import matplotlib.pyplot as plt
    from eqcorrscan.utils import plotting
    # Check that we actually have the correct stations
    realstations = []
    for station in stations:
        st = stream.select(station=station)
        if st:
            realstations += station
    del st
    stream_copy = stream.copy()
    # Force convert to int16
    for tr in stream_copy:
        # int16 max range is +/- 32767
        if max(abs(tr.data)) > 32767:
            tr.data = 32767 * (tr.data / max(abs(tr.data)))
            # Make sure that the data aren't clipped it they are high gain
            # scale the data
        tr.data = tr.data.astype(np.int16)
    # The internal _node_loop converts energy to int16 too to converse memory,
    # to do this it forces the maximum of a single energy trace to be 500 and
    # normalises to this level - this only works for fewer than 65 channels of
    # data
    if len(stream_copy) > 130:
        raise OverflowError('Too many streams, either re-code and cope with' +
                            'either more memory usage, or less precision, or' +
                            'reduce data volume')
    detections = []
    detect_lags = []
    parallel = True
    plotvar = True
    mem_issue = False
    # Loop through each node in the input
    # Linear run
    print('Computing the energy stacks')
    if not parallel:
        for i in range(0, len(nodes)):
            print(i)
            if not mem_issue:
                j, a = _node_loop(stations, lags[:, i], stream, plot=True)
                if 'energy' not in locals():
                    energy = a
                else:
                    energy = np.concatenate((energy, a), axis=0)
                print('energy: ' + str(np.shape(energy)))
            else:
                j, filename = _node_loop(stations, lags[:, i], stream, i,
                                         mem_issue)
        energy = np.array(energy)
        print(np.shape(energy))
    else:
        # Parallel run
        num_cores = cores
        if num_cores > len(nodes):
            num_cores = len(nodes)
        if num_cores > cpu_count():
            num_cores = cpu_count()
        pool = Pool(processes=num_cores)
        results = [pool.apply_async(_node_loop, args=(stations, lags[:, i],
                                                      stream, i, clip_level,
                                                      mem_issue, instance))
                   for i in range(len(nodes))]
        pool.close()
        if not mem_issue:
            print('Computing the cumulative network response from memory')
            energy = [p.get() for p in results]
            pool.join()
            energy.sort(key=lambda tup: tup[0])
            energy = [node[1] for node in energy]
            energy = np.concatenate(energy, axis=0)
            print(energy.shape)
        else:
            pool.join()
    # Now compute the cumulative network response and then detect possible
    # events
    if not mem_issue:
        print(energy.shape)
        indeces = np.argmax(energy, axis=0)  # Indeces of maximum energy
        print(indeces.shape)
        cum_net_resp = np.array([np.nan] * len(indeces))
        cum_net_resp[0] = energy[indeces[0]][0]
        peak_nodes = [nodes[indeces[0]]]
        for i in range(1, len(indeces)):
            cum_net_resp[i] = energy[indeces[i]][i]
            peak_nodes.append(nodes[indeces[i]])
        del energy, indeces
    else:
        print('Reading the temp files and computing network response')
        node_splits = int(len(nodes) // num_cores)
        indeces = [range(node_splits)]
        for i in range(1, num_cores - 1):
            indeces.append(range(node_splits * i, node_splits * (i + 1)))
        indeces.append(range(node_splits * (i + 1), len(nodes)))
        pool = Pool(processes=num_cores)
        results = [pool.apply_async(_cum_net_resp, args=(indeces[i], instance))
                   for i in range(num_cores)]
        pool.close()
        results = [p.get() for p in results]
        pool.join()
        responses = [result[0] for result in results]
        print(np.shape(responses))
        node_indeces = [result[1] for result in results]
        cum_net_resp = np.array(responses)
        indeces = np.argmax(cum_net_resp, axis=0)
        print(indeces.shape)
        print(cum_net_resp.shape)
        cum_net_resp = np.array([cum_net_resp[indeces[i]][i]
                                 for i in range(len(indeces))])
        peak_nodes = [nodes[node_indeces[indeces[i]][i]]
                      for i in range(len(indeces))]
        del indeces, node_indeces
    if plotvar:
        cum_net_trace = deepcopy(stream[0])
        cum_net_trace.data = cum_net_resp
        cum_net_trace.stats.station = 'NR'
        cum_net_trace.stats.channel = ''
        cum_net_trace.stats.network = 'Z'
        cum_net_trace.stats.location = ''
        cum_net_trace.stats.starttime = stream[0].stats.starttime
        cum_net_trace = obspy.Stream(cum_net_trace)
        cum_net_trace += stream.select(channel='*N')
        cum_net_trace += stream.select(channel='*1')
        cum_net_trace.sort(['network', 'station', 'channel'])
        # np.save('cum_net_resp.npy',cum_net_resp)
        #     cum_net_trace.plot(size=(800,600), equal_scale=False,\
        #                        outfile='NR_timeseries.eps')

    # Find detection within this network response
    print('Finding detections in the cumulatve network response')
    detections = _find_detections(cum_net_resp, peak_nodes, threshold,
                                  thresh_type, stream[0].stats.sampling_rate,
                                  realstations, gap)
    del cum_net_resp
    templates = []
    nodesout = []
    good_detections = []
    if detections:
        print('Converting detections in to templates')
        # Generate a catalog of detections
        detections_cat = Catalog()
        for j, detection in enumerate(detections):
            print('Converting for detection ' + str(j) + ' of ' +
                  str(len(detections)))
            # Create an event for each detection
            event = Event()
            # Set up some header info for the event
            event.event_descriptions.append(EventDescription())
            event.event_descriptions[0].text = 'Brightness detection'
            event.creation_info = CreationInfo(agency_id='EQcorrscan')
            copy_of_stream = deepcopy(stream_copy)
            # Convert detections to obspy.core.event type -
            # name of detection template is the node.
            node = (detection.template_name.split('_')[0],
                    detection.template_name.split('_')[1],
                    detection.template_name.split('_')[2])
            print(node)
            # Look up node in nodes and find the associated lags
            index = nodes.index(node)
            detect_lags = lags[:, index]
            ksta = Comment(text='Number of stations=' + len(detect_lags))
            event.origins.append(Origin())
            event.origins[0].comments.append(ksta)
            event.origins[0].time = copy_of_stream[0].stats.starttime +\
                detect_lags[0] + detection.detect_time
            event.origins[0].latitude = node[0]
            event.origins[0].longitude = node[1]
            event.origins[0].depth = node[2]
            for i, detect_lag in enumerate(detect_lags):
                station = stations[i]
                st = copy_of_stream.select(station=station)
                if len(st) != 0:
                    for tr in st:
                        _waveform_id = WaveformStreamID(station_code=tr.stats.
                                                        station,
                                                        channel_code=tr.stats.
                                                        channel,
                                                        network_code='NA')
                        event.picks.append(Pick(waveform_id=_waveform_id,
                                                time=tr.stats.starttime +
                                                detect_lag +
                                                detection.detect_time +
                                                pre_pick,
                                                onset='emergent',
                                                evalutation_mode='automatic'))
            print('Generating template for detection: ' + str(j))
            template = (_template_gen(event.picks, copy_of_stream,
                        template_length, 'all'))
            template_name = template_saveloc + '/' +\
                str(template[0].stats.starttime) + '.ms'
            # In the interests of RAM conservation we write then read
            # Check coherancy here!
            temp_coher, kchan = coherence(template, coherence_stations,
                                          coherence_clip)
            coh_thresh = float(coherence_thresh[0]) - kchan / \
                float(coherence_thresh[1])
            if temp_coher > coh_thresh:
                template.write(template_name, format="MSEED")
                print('Written template as: ' + template_name)
                print('---------------------------------coherence LEVEL: ' +
                      str(temp_coher))
                coherant = True
            else:
                print('Template was incoherant, coherence level: ' +
                      str(temp_coher))
                coherant = False
            del copy_of_stream, tr, template
            if coherant:
                templates.append(obsread(template_name))
                nodesout += [node]
                good_detections.append(detection)
            else:
                print('No template for you')
    if plotvar:
        all_detections = [(cum_net_trace[-1].stats.starttime +
                           detection.detect_time).datetime
                          for detection in detections]
        good_detections = [(cum_net_trace[-1].stats.starttime +
                            detection.detect_time).datetime
                           for detection in good_detections]
        if not plotsave:
            plotting.NR_plot(cum_net_trace[0:-1],
                             obspy.Stream(cum_net_trace[-1]),
                             detections=good_detections,
                             size=(18.5, 10),
                             title='Network response')
            # cum_net_trace.plot(size=(800,600), equal_scale=False)
        else:
            savefile = 'plots/' +\
                cum_net_trace[0].stats.starttime.datetime.strftime('%Y%m%d') +\
                '_NR_timeseries.pdf'
            plotting.NR_plot(cum_net_trace[0:-1],
                             obspy.Stream(cum_net_trace[-1]),
                             detections=good_detections,
                             size=(18.5, 10), save=savefile,
                             title='Network response')
    nodesout = list(set(nodesout))
    return templates, nodesout
예제 #12
0
def brightness(
    stations,
    nodes,
    lags,
    stream,
    threshold,
    thresh_type,
    template_length,
    template_saveloc,
    coherence_thresh,
    coherence_stations=["all"],
    coherence_clip=False,
    gap=2.0,
    clip_level=100,
    instance=0,
    pre_pick=0.2,
    plotsave=True,
    cores=1,
):
    r"""Function to calculate the brightness function in terms of energy for\
    a day of data over the entire network for a given grid of nodes.

    Note data in stream must be all of the same length and have the same
    sampling rates.

    :type stations: list
    :param stations: List of station names from in the form where stations[i]\
    refers to nodes[i][:] and lags[i][:]
    :type nodes: list, tuple
    :param nodes: List of node points where nodes[i] referes to stations[i]\
    and nodes[:][:][0] is latitude in degrees, nodes[:][:][1] is longitude in\
    degrees, nodes[:][:][2] is depth in km.
    :type lags: :class: 'numpy.array'
    :param lags: Array of arrays where lags[i][:] refers to stations[i].\
    lags[i][j] should be the delay to the nodes[i][j] for stations[i] in\
    seconds.
    :type stream: :class: `obspy.Stream`
    :param data: Data through which to look for detections.
    :type threshold: float
    :param threshold: Threshold value for detection of template within the\
    brightness function
    :type thresh_type: str
    :param thresh_type: Either MAD or abs where MAD is the Median Absolute\
    Deviation and abs is an absoulte brightness.
    :type template_length: float
    :param template_length: Length of template to extract in seconds
    :type template_saveloc: str
    :param template_saveloc: Path of where to save the templates.
    :type coherence_thresh: tuple of floats
    :param coherence_thresh: Threshold for removing incoherant peaks in the\
            network response, those below this will not be used as templates.\
            Must be in the form of (a,b) where the coherence is given by:\
            a-kchan/b where kchan is the number of channels used to compute\
            the coherence
    :type coherence_stations: list
    :param coherence_stations: List of stations to use in the coherance\
            thresholding - defaults to 'all' which uses all the stations.
    :type coherence_clip: float
    :param coherence_clip: tuple
    :type coherence_clip: Start and end in seconds of data to window around,\
            defaults to False, which uses all the data given.
    :type pre_pick: float
    :param pre_pick: Seconds before the detection time to include in template
    :type plotsave: bool
    :param plotsave: Save or show plots, if False will try and show the plots\
            on screen - as this is designed for bulk use this is set to\
            True to save any plots rather than show them if you create\
            them - changes the backend of matplotlib, so if is set to\
            False you will see NO PLOTS!
    :type cores: int
    :param core: Number of cores to use, defaults to 1.
    :type clip_level: float
    :param clip_level: Multiplier applied to the mean deviation of the energy\
                    as an upper limit, used to remove spikes (earthquakes, \
                    lightning, electircal spikes) from the energy stack.
    :type gap: float
    :param gap: Minimum inter-event time in seconds for detections

    :return: list of templates as :class: `obspy.Stream` objects
    """
    from eqcorrscan.core.template_gen import _template_gen

    if plotsave:
        import matplotlib

        matplotlib.use("Agg")
        import matplotlib.pyplot as plt

        plt.ioff()
    # from joblib import Parallel, delayed
    from multiprocessing import Pool, cpu_count
    from eqcorrscan.utils.Sfile_util import PICK
    from copy import deepcopy
    from obspy import read as obsread
    import obspy.Stream
    import matplotlib.pyplot as plt
    from eqcorrscan.utils import EQcorrscan_plotting as plotting

    # Check that we actually have the correct stations
    realstations = []
    for station in stations:
        st = stream.select(station=station)
        if st:
            realstations += station
    del st
    stream_copy = stream.copy()
    # Force convert to int16
    for tr in stream_copy:
        # int16 max range is +/- 32767
        if max(abs(tr.data)) > 32767:
            tr.data = 32767 * (tr.data / max(abs(tr.data)))
            # Make sure that the data aren't clipped it they are high gain
            # scale the data
        tr.data = tr.data.astype(np.int16)
    # The internal _node_loop converts energy to int16 too to converse memory,
    # to do this it forces the maximum of a single energy trace to be 500 and
    # normalises to this level - this only works for fewer than 65 channels of
    # data
    if len(stream_copy) > 130:
        raise OverflowError(
            "Too many streams, either re-code and cope with"
            + "either more memory usage, or less precision, or"
            + "reduce data volume"
        )
    detections = []
    detect_lags = []
    parallel = True
    plotvar = True
    mem_issue = False
    # Loop through each node in the input
    # Linear run
    print "Computing the energy stacks"
    if not parallel:
        for i in range(0, len(nodes)):
            print i
            if not mem_issue:
                j, a = _node_loop(stations, lags[:, i], stream, plot=True)
                if "energy" not in locals():
                    energy = a
                else:
                    energy = np.concatenate((energy, a), axis=0)
                print "energy: " + str(np.shape(energy))
            else:
                j, filename = _node_loop(stations, lags[:, i], stream, i, mem_issue)
        energy = np.array(energy)
        print np.shape(energy)
    else:
        # Parallel run
        num_cores = cores
        if num_cores > len(nodes):
            num_cores = len(nodes)
        if num_cores > cpu_count():
            num_cores = cpu_count()
        pool = Pool(processes=num_cores, maxtasksperchild=None)
        results = [
            pool.apply_async(_node_loop, args=(stations, lags[:, i], stream, i, clip_level, mem_issue, instance))
            for i in range(len(nodes))
        ]
        pool.close()
        if not mem_issue:
            print "Computing the cumulative network response from memory"
            energy = [p.get() for p in results]
            pool.join()
            energy.sort(key=lambda tup: tup[0])
            energy = [node[1] for node in energy]
            energy = np.concatenate(energy, axis=0)
            print energy.shape
        else:
            pool.join()
    # Now compute the cumulative network response and then detect possible
    # events
    if not mem_issue:
        print energy.shape
        indeces = np.argmax(energy, axis=0)  # Indeces of maximum energy
        print indeces.shape
        cum_net_resp = np.array([np.nan] * len(indeces))
        cum_net_resp[0] = energy[indeces[0]][0]
        peak_nodes = [nodes[indeces[0]]]
        for i in range(1, len(indeces)):
            cum_net_resp[i] = energy[indeces[i]][i]
            peak_nodes.append(nodes[indeces[i]])
        del energy, indeces
    else:
        print "Reading the temp files and computing network response"
        node_splits = len(nodes) / num_cores
        indeces = [range(node_splits)]
        for i in range(1, num_cores - 1):
            indeces.append(range(node_splits * i, node_splits * (i + 1)))
        indeces.append(range(node_splits * (i + 1), len(nodes)))
        pool = Pool(processes=num_cores, maxtasksperchild=None)
        results = [pool.apply_async(_cum_net_resp, args=(indeces[i], instance)) for i in range(num_cores)]
        pool.close()
        results = [p.get() for p in results]
        pool.join()
        responses = [result[0] for result in results]
        print np.shape(responses)
        node_indeces = [result[1] for result in results]
        cum_net_resp = np.array(responses)
        indeces = np.argmax(cum_net_resp, axis=0)
        print indeces.shape
        print cum_net_resp.shape
        cum_net_resp = np.array([cum_net_resp[indeces[i]][i] for i in range(len(indeces))])
        peak_nodes = [nodes[node_indeces[indeces[i]][i]] for i in range(len(indeces))]
        del indeces, node_indeces
    if plotvar:
        cum_net_trace = deepcopy(stream[0])
        cum_net_trace.data = cum_net_resp
        cum_net_trace.stats.station = "NR"
        cum_net_trace.stats.channel = ""
        cum_net_trace.stats.network = "Z"
        cum_net_trace.stats.location = ""
        cum_net_trace.stats.starttime = stream[0].stats.starttime
        cum_net_trace = obspy.Stream(cum_net_trace)
        cum_net_trace += stream.select(channel="*N")
        cum_net_trace += stream.select(channel="*1")
        cum_net_trace.sort(["network", "station", "channel"])
        # np.save('cum_net_resp.npy',cum_net_resp)
        #     cum_net_trace.plot(size=(800,600), equal_scale=False,\
        #                        outfile='NR_timeseries.eps')

    # Find detection within this network response
    print "Finding detections in the cumulatve network response"
    detections = _find_detections(
        cum_net_resp, peak_nodes, threshold, thresh_type, stream[0].stats.sampling_rate, realstations, gap
    )
    del cum_net_resp
    templates = []
    nodesout = []
    good_detections = []
    if detections:
        print "Converting detections in to templates"
        for j, detection in enumerate(detections):
            print "Converting for detection " + str(j) + " of " + str(len(detections))
            copy_of_stream = deepcopy(stream_copy)
            # Convert detections to PICK type - name of detection template
            # is the node.
            node = (
                detection.template_name.split("_")[0],
                detection.template_name.split("_")[1],
                detection.template_name.split("_")[2],
            )
            print node
            # Look up node in nodes and find the associated lags
            index = nodes.index(node)
            detect_lags = lags[:, index]
            picks = []
            for i, detect_lag in enumerate(detect_lags):
                station = stations[i]
                st = copy_of_stream.select(station=station)
                if len(st) != 0:
                    for tr in st:
                        picks.append(
                            PICK(
                                station=station,
                                channel=tr.stats.channel,
                                impulsivity="E",
                                phase="S",
                                weight="3",
                                polarity="",
                                time=tr.stats.starttime + detect_lag + detection.detect_time - pre_pick,
                                coda="",
                                amplitude="",
                                peri="",
                                azimuth="",
                                velocity="",
                                AIN="",
                                SNR="",
                                azimuthres="",
                                timeres="",
                                finalweight="",
                                distance="",
                                CAZ="",
                            )
                        )
            print "Generating template for detection: " + str(j)
            template = _template_gen(picks, copy_of_stream, template_length, "all")
            template_name = template_saveloc + "/" + str(template[0].stats.starttime) + ".ms"
            # In the interests of RAM conservation we write then read
            # Check coherancy here!
            temp_coher, kchan = coherence(template, coherence_stations, coherence_clip)
            coh_thresh = float(coherence_thresh[0]) - kchan / float(coherence_thresh[1])
            if temp_coher > coh_thresh:
                template.write(template_name, format="MSEED")
                print "Written template as: " + template_name
                print "---------------------------------coherence LEVEL: " + str(temp_coher)
                coherant = True
            else:
                print "Template was incoherant, coherence level: " + str(temp_coher)
                coherant = False
            del copy_of_stream, tr, template
            if coherant:
                templates.append(obsread(template_name))
                nodesout += [node]
                good_detections.append(detection)
            else:
                print "No template for you"
    if plotvar:
        all_detections = [
            (cum_net_trace[-1].stats.starttime + detection.detect_time).datetime for detection in detections
        ]
        good_detections = [
            (cum_net_trace[-1].stats.starttime + detection.detect_time).datetime for detection in good_detections
        ]
        if not plotsave:
            plotting.NR_plot(
                cum_net_trace[0:-1],
                obspy.Stream(cum_net_trace[-1]),
                detections=good_detections,
                size=(18.5, 10),
                title="Network response",
            )
            # cum_net_trace.plot(size=(800,600), equal_scale=False)
        else:
            savefile = "plots/" + cum_net_trace[0].stats.starttime.datetime.strftime("%Y%m%d") + "_NR_timeseries.pdf"
            plotting.NR_plot(
                cum_net_trace[0:-1],
                obspy.Stream(cum_net_trace[-1]),
                detections=good_detections,
                size=(18.5, 10),
                save=savefile,
                title="Network response",
            )
    nodesout = list(set(nodesout))
    return templates, nodesout
예제 #13
0
 def test_no_data_for_channel(self):
     st = self.st.copy()
     st.select(station='LABE', channel='SN')[0].stats.starttime += 2000
     template = _template_gen(self.picks, st, 10)
     self.assertEqual(len(template), 10)
예제 #14
0
 def test_undefined_phase_type(self):
     with self.assertRaises(AssertionError):
         _template_gen(picks=self.picks,
                       st=self.st.copy(),
                       length=2,
                       swin='bob')
예제 #15
0
 def test_misc(self):
     template = _template_gen(self.picks, self.st.copy(), 10)
     self.assertEqual(len(template), len(self.picks))
예제 #16
0
    call('NLLoc /Users/home/hoppche/NLLoc/mrp/run/nlloc_mrp.in', shell=True)
    # Now reading NLLoc output back into catalog as new origin
    out_w_ext = glob(outfile + '*.grid0.loc.hyp')
    new_o = read_nlloc_hyp(out_w_ext[0],
                           coordinate_converter=my_conversion,
                           picks=ev.picks)
    ev.origins.append(new_o[0].origins[0])
    ev.preferred_origin_id = str(new_o[0].origins[0].resource_id)

# Cut templates for each new event based on new picks
for event in refined_cat:
    ev_name = str(event.resource_id).split('/')[2]
    st = template_dict[event.resource_id]
    st1 = pre_processing.shortproc(st,
                                   lowcut=1.0,
                                   highcut=20.0,
                                   filt_order=3,
                                   samp_rate=50,
                                   debug=0)
    print('Feeding stream to _template_gen...')
    template = template_gen._template_gen(event.picks,
                                          st1,
                                          length=4.0,
                                          swin='all',
                                          prepick=0.5)
    print('Writing event ' + ev_name + ' to file...')
    template.write('/media/chet/hdd/seismic/NZ/templates/rotnga_2015/' +
                   'refined_picks/' + ev_name + '_50Hz.mseed',
                   format="MSEED")
    del st, st1, template
예제 #17
0
                continue
            else:
                print('Merging stream...')
                st.merge(fill_value='interpolate')
                day_st = st.copy()
                for event in day_cat:
                    ev_name = str(event.resource_id).split('/')[2]
                    origin_time = event.origins[0].time
                    print('Trimming data around event time...')
                    day_st.trim(origin_time - 120, origin_time + 120)
                    print('Preprocessing data for day: ' + str(starttime.date))
                    temp_st = pre_processing.shortproc(day_st,
                                                       lowcut=1.0,
                                                       highcut=20.0,
                                                       filt_order=3,
                                                       samp_rate=100,
                                                       debug=0)
                    del day_st
                    print('Feeding stream to _template_gen...')
                    template = _template_gen(event.picks,
                                             temp_st,
                                             length=4,
                                             swin='all',
                                             prepick=0.5)
                    print('Writing event ' + ev_name + ' to file...')
                    template.write('/media/rotnga_data/templates/2015/' +
                                   ev_name + '.mseed',
                                   format="MSEED")
                    del temp_st, template
                del day_cat
예제 #18
0
        print ' '.join(['Reading data from', wavefile])
        if 'st' not in locals():
            st = read(wavefile)
        else:
            st += read(wavefile)

    st = st.merge(fill_value='interpolate')
    day = st[0].stats.starttime.date

    # Process the data with our required parameters
    for tr in st:
        tr = pre_processing.dayproc(tr, 1.0, 20.0, 3, 100.0,\
                                    debug, day)

    # Use the template generation function to cut our templates
    template = template_gen._template_gen(picks, st, length=1.0, swin='all',
                                          prepick=0.1, plot=True)
    # This will generate an obspy.Stream object
    # Append this Stream to the list of templates
    templates += [template]
    template_names.append('_'.join(['tutorial', str(i)]))

    # Save template for later
    template.write(os.path.join(data_directory, '_'.join([template_names[i],
                                                          'template.ms'])),
                   format='MSEED')
    # Delete excess information from memory If you are re-using this script
    # with the same templates you should be able to comment out this loop
    # once you have generated your templates once.
    del template, st

# Extract the stations from the templates
예제 #19
0
 def test_snr_cutoff(self):
     template = _template_gen(self.picks, self.st.copy(), 10, min_snr=100)
     self.assertEqual(len(template), 1)
예제 #20
0
 def test_missing_data(self):
     picks = copy.deepcopy(self.picks)
     picks.append(picks[-1])
     picks[-1].waveform_id.station_code = 'DUMMY'
     template = _template_gen(picks, self.st.copy(), 10)
     self.assertFalse('DUMMY' in [tr.stats.station for tr in template])
예제 #21
0
    wavefiles=list(set(wavefiles))
    for wavefile in wavefiles:
        print 'Reading data from '+wavefile
        if not 'st' in locals():
            st=read(wavefile)
        else:
            st+=read(wavefile)
    st=st.merge(fill_value='interpolate')
    day=st[0].stats.starttime.date
    for tr in st:
        tr=pre_processing.dayproc(tr, 1.0, 20.0, 3, 100.0,\
                                  matchdef.debug, day)
    # Apply a small amoutn of delay before the pick
    for pick in picks:
        pick.time=pick.time-0.1
    template=template_gen._template_gen(picks, st, 1.0, 'all',\
                                        tempdef=templatedef)
    # This will generate an obspy.Stream object
    # Append this Stream to the list of templates
    templates+=[template]
    template_names.append('tutorial_'+str(i))
    # Plot the template just to check that all is well!
    template.plot(size=(800,600), equal_scale=False)
    # Save template for later
    template.write('test_data/tutorial_data/'+template_names[i]+'_template.ms',\
                   format='MSEED')
    i+=1
    del template, st

# Extract the stations from the templates
for template in templates:
    if not 'stachans' in locals():
예제 #22
0
 def test_no_matched_picks(self):
     picks = [copy.deepcopy(self.picks[0])]
     picks[0].waveform_id.station_code = 'DUMMY'
     template = _template_gen(picks, self.st.copy(), 10)
     self.assertFalse(template)
예제 #23
0
            st = read(wavefile)
        else:
            st += read(wavefile)

    st = st.merge(fill_value='interpolate')
    day = st[0].stats.starttime.date

    # Process the data with our required parameters
    for tr in st:
        tr = pre_processing.dayproc(tr, 1.0, 20.0, 3, 100.0,\
                                    debug, day)

    # Use the template generation function to cut our templates
    template = template_gen._template_gen(picks,
                                          st,
                                          length=1.0,
                                          swin='all',
                                          prepick=0.1,
                                          plot=True)
    # This will generate an obspy.Stream object
    # Append this Stream to the list of templates
    templates += [template]
    template_names.append('_'.join(['tutorial', str(i)]))

    # Save template for later
    template.write(os.path.join(data_directory,
                                '_'.join([template_names[i], 'template.ms'])),
                   format='MSEED')
    # Delete excess information from memory If you are re-using this script
    # with the same templates you should be able to comment out this loop
    # once you have generated your templates once.
    del template, st
예제 #24
0
 def test_debug_levels(self):
     print(len(self.picks))
     print(len(self.st))
     template = _template_gen(self.picks, self.st.copy(), 10, debug=3)
     self.assertEqual(len(template), len(self.picks))
예제 #25
0
def brightness(stations,
               nodes,
               lags,
               stream,
               threshold,
               thresh_type,
               template_length,
               template_saveloc,
               coherence_thresh,
               coherence_stations=['all'],
               coherence_clip=False,
               gap=2.0,
               clip_level=100,
               instance=0,
               pre_pick=0.2,
               plotsave=True,
               cores=1):
    r"""Function to calculate the brightness function in terms of energy for \
    a day of data over the entire network for a given grid of nodes.

    Note data in stream must be all of the same length and have the same
    sampling rates.

    :type stations: list
    :param stations: List of station names from in the form where stations[i] \
        refers to nodes[i][:] and lags[i][:]
    :type nodes: list, tuple
    :param nodes: List of node points where nodes[i] referes to stations[i] \
        and nodes[:][:][0] is latitude in degrees, nodes[:][:][1] is \
        longitude in degrees, nodes[:][:][2] is depth in km.
    :type lags: :class: 'numpy.array'
    :param lags: Array of arrays where lags[i][:] refers to stations[i]. \
        lags[i][j] should be the delay to the nodes[i][j] for stations[i] in \
        seconds.
    :type stream: :class: `obspy.Stream`
    :param data: Data through which to look for detections.
    :type threshold: float
    :param threshold: Threshold value for detection of template within the \
        brightness function
    :type thresh_type: str
    :param thresh_type: Either MAD or abs where MAD is the Median Absolute \
        Deviation and abs is an absoulte brightness.
    :type template_length: float
    :param template_length: Length of template to extract in seconds
    :type template_saveloc: str
    :param template_saveloc: Path of where to save the templates.
    :type coherence_thresh: tuple of floats
    :param coherence_thresh: Threshold for removing incoherant peaks in the \
            network response, those below this will not be used as templates. \
            Must be in the form of (a,b) where the coherence is given by: \
            a-kchan/b where kchan is the number of channels used to compute \
            the coherence
    :type coherence_stations: list
    :param coherence_stations: List of stations to use in the coherance \
            thresholding - defaults to 'all' which uses all the stations.
    :type coherence_clip: float
    :param coherence_clip: tuple
    :type coherence_clip: Start and end in seconds of data to window around, \
            defaults to False, which uses all the data given.
    :type pre_pick: float
    :param pre_pick: Seconds before the detection time to include in template
    :type plotsave: bool
    :param plotsave: Save or show plots, if False will try and show the plots \
            on screen - as this is designed for bulk use this is set to \
            True to save any plots rather than show them if you create \
            them - changes the backend of matplotlib, so if is set to \
            False you will see NO PLOTS!
    :type cores: int
    :param core: Number of cores to use, defaults to 1.
    :type clip_level: float
    :param clip_level: Multiplier applied to the mean deviation of the energy \
                    as an upper limit, used to remove spikes (earthquakes, \
                    lightning, electircal spikes) from the energy stack.
    :type gap: float
    :param gap: Minimum inter-event time in seconds for detections

    :return: list of templates as :class: `obspy.Stream` objects
    """
    from eqcorrscan.core.template_gen import _template_gen
    if plotsave:
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
        plt.ioff()
    # from joblib import Parallel, delayed
    from multiprocessing import Pool, cpu_count
    from copy import deepcopy
    from obspy import read as obsread
    from obspy.core.event import Catalog, Event, Pick, WaveformStreamID, Origin
    from obspy.core.event import EventDescription, CreationInfo, Comment
    import obspy.Stream
    import matplotlib.pyplot as plt
    from eqcorrscan.utils import EQcorrscan_plotting as plotting
    # Check that we actually have the correct stations
    realstations = []
    for station in stations:
        st = stream.select(station=station)
        if st:
            realstations += station
    del st
    stream_copy = stream.copy()
    # Force convert to int16
    for tr in stream_copy:
        # int16 max range is +/- 32767
        if max(abs(tr.data)) > 32767:
            tr.data = 32767 * (tr.data / max(abs(tr.data)))
            # Make sure that the data aren't clipped it they are high gain
            # scale the data
        tr.data = tr.data.astype(np.int16)
    # The internal _node_loop converts energy to int16 too to converse memory,
    # to do this it forces the maximum of a single energy trace to be 500 and
    # normalises to this level - this only works for fewer than 65 channels of
    # data
    if len(stream_copy) > 130:
        raise OverflowError('Too many streams, either re-code and cope with' +
                            'either more memory usage, or less precision, or' +
                            'reduce data volume')
    detections = []
    detect_lags = []
    parallel = True
    plotvar = True
    mem_issue = False
    # Loop through each node in the input
    # Linear run
    print('Computing the energy stacks')
    if not parallel:
        for i in range(0, len(nodes)):
            print(i)
            if not mem_issue:
                j, a = _node_loop(stations, lags[:, i], stream, plot=True)
                if 'energy' not in locals():
                    energy = a
                else:
                    energy = np.concatenate((energy, a), axis=0)
                print('energy: ' + str(np.shape(energy)))
            else:
                j, filename = _node_loop(stations, lags[:, i], stream, i,
                                         mem_issue)
        energy = np.array(energy)
        print(np.shape(energy))
    else:
        # Parallel run
        num_cores = cores
        if num_cores > len(nodes):
            num_cores = len(nodes)
        if num_cores > cpu_count():
            num_cores = cpu_count()
        pool = Pool(processes=num_cores)
        results = [
            pool.apply_async(_node_loop,
                             args=(stations, lags[:, i], stream, i, clip_level,
                                   mem_issue, instance))
            for i in range(len(nodes))
        ]
        pool.close()
        if not mem_issue:
            print('Computing the cumulative network response from memory')
            energy = [p.get() for p in results]
            pool.join()
            energy.sort(key=lambda tup: tup[0])
            energy = [node[1] for node in energy]
            energy = np.concatenate(energy, axis=0)
            print(energy.shape)
        else:
            pool.join()
    # Now compute the cumulative network response and then detect possible
    # events
    if not mem_issue:
        print(energy.shape)
        indeces = np.argmax(energy, axis=0)  # Indeces of maximum energy
        print(indeces.shape)
        cum_net_resp = np.array([np.nan] * len(indeces))
        cum_net_resp[0] = energy[indeces[0]][0]
        peak_nodes = [nodes[indeces[0]]]
        for i in range(1, len(indeces)):
            cum_net_resp[i] = energy[indeces[i]][i]
            peak_nodes.append(nodes[indeces[i]])
        del energy, indeces
    else:
        print('Reading the temp files and computing network response')
        node_splits = len(nodes) // num_cores
        indeces = [range(node_splits)]
        for i in range(1, num_cores - 1):
            indeces.append(range(node_splits * i, node_splits * (i + 1)))
        indeces.append(range(node_splits * (i + 1), len(nodes)))
        pool = Pool(processes=num_cores)
        results = [
            pool.apply_async(_cum_net_resp, args=(indeces[i], instance))
            for i in range(num_cores)
        ]
        pool.close()
        results = [p.get() for p in results]
        pool.join()
        responses = [result[0] for result in results]
        print(np.shape(responses))
        node_indeces = [result[1] for result in results]
        cum_net_resp = np.array(responses)
        indeces = np.argmax(cum_net_resp, axis=0)
        print(indeces.shape)
        print(cum_net_resp.shape)
        cum_net_resp = np.array(
            [cum_net_resp[indeces[i]][i] for i in range(len(indeces))])
        peak_nodes = [
            nodes[node_indeces[indeces[i]][i]] for i in range(len(indeces))
        ]
        del indeces, node_indeces
    if plotvar:
        cum_net_trace = deepcopy(stream[0])
        cum_net_trace.data = cum_net_resp
        cum_net_trace.stats.station = 'NR'
        cum_net_trace.stats.channel = ''
        cum_net_trace.stats.network = 'Z'
        cum_net_trace.stats.location = ''
        cum_net_trace.stats.starttime = stream[0].stats.starttime
        cum_net_trace = obspy.Stream(cum_net_trace)
        cum_net_trace += stream.select(channel='*N')
        cum_net_trace += stream.select(channel='*1')
        cum_net_trace.sort(['network', 'station', 'channel'])
        # np.save('cum_net_resp.npy',cum_net_resp)
        #     cum_net_trace.plot(size=(800,600), equal_scale=False,\
        #                        outfile='NR_timeseries.eps')

    # Find detection within this network response
    print('Finding detections in the cumulatve network response')
    detections = _find_detections(cum_net_resp, peak_nodes, threshold,
                                  thresh_type, stream[0].stats.sampling_rate,
                                  realstations, gap)
    del cum_net_resp
    templates = []
    nodesout = []
    good_detections = []
    if detections:
        print('Converting detections in to templates')
        # Generate a catalog of detections
        detections_cat = Catalog()
        for j, detection in enumerate(detections):
            print('Converting for detection ' + str(j) + ' of ' +
                  str(len(detections)))
            # Create an event for each detection
            event = Event()
            # Set up some header info for the event
            event.event_descriptions.append(EventDescription())
            event.event_descriptions[0].text = 'Brightness detection'
            event.creation_info = CreationInfo(agency_id='EQcorrscan')
            copy_of_stream = deepcopy(stream_copy)
            # Convert detections to obspy.core.event type -
            # name of detection template is the node.
            node = (detection.template_name.split('_')[0],
                    detection.template_name.split('_')[1],
                    detection.template_name.split('_')[2])
            print(node)
            # Look up node in nodes and find the associated lags
            index = nodes.index(node)
            detect_lags = lags[:, index]
            ksta = Comment(text='Number of stations=' + len(detect_lags))
            event.origins.append(Origin())
            event.origins[0].comments.append(ksta)
            event.origins[0].time = copy_of_stream[0].stats.starttime +\
                detect_lags[0] + detection.detect_time
            event.origins[0].latitude = node[0]
            event.origins[0].longitude = node[1]
            event.origins[0].depth = node[2]
            for i, detect_lag in enumerate(detect_lags):
                station = stations[i]
                st = copy_of_stream.select(station=station)
                if len(st) != 0:
                    for tr in st:
                        _waveform_id = WaveformStreamID(
                            station_code=tr.stats.station,
                            channel_code=tr.stats.channel,
                            network_code='NA')
                        event.picks.append(
                            Pick(waveform_id=_waveform_id,
                                 time=tr.stats.starttime + detect_lag +
                                 detection.detect_time + pre_pick,
                                 onset='emergent',
                                 evalutation_mode='automatic'))
            print('Generating template for detection: ' + str(j))
            template = (_template_gen(event.picks, copy_of_stream,
                                      template_length, 'all'))
            template_name = template_saveloc + '/' +\
                str(template[0].stats.starttime) + '.ms'
            # In the interests of RAM conservation we write then read
            # Check coherancy here!
            temp_coher, kchan = coherence(template, coherence_stations,
                                          coherence_clip)
            coh_thresh = float(coherence_thresh[0]) - kchan / \
                float(coherence_thresh[1])
            if temp_coher > coh_thresh:
                template.write(template_name, format="MSEED")
                print('Written template as: ' + template_name)
                print('---------------------------------coherence LEVEL: ' +
                      str(temp_coher))
                coherant = True
            else:
                print('Template was incoherant, coherence level: ' +
                      str(temp_coher))
                coherant = False
            del copy_of_stream, tr, template
            if coherant:
                templates.append(obsread(template_name))
                nodesout += [node]
                good_detections.append(detection)
            else:
                print('No template for you')
    if plotvar:
        all_detections = [(cum_net_trace[-1].stats.starttime +
                           detection.detect_time).datetime
                          for detection in detections]
        good_detections = [(cum_net_trace[-1].stats.starttime +
                            detection.detect_time).datetime
                           for detection in good_detections]
        if not plotsave:
            plotting.NR_plot(cum_net_trace[0:-1],
                             obspy.Stream(cum_net_trace[-1]),
                             detections=good_detections,
                             size=(18.5, 10),
                             title='Network response')
            # cum_net_trace.plot(size=(800,600), equal_scale=False)
        else:
            savefile = 'plots/' +\
                cum_net_trace[0].stats.starttime.datetime.strftime('%Y%m%d') +\
                '_NR_timeseries.pdf'
            plotting.NR_plot(cum_net_trace[0:-1],
                             obspy.Stream(cum_net_trace[-1]),
                             detections=good_detections,
                             size=(18.5, 10),
                             save=savefile,
                             title='Network response')
    nodesout = list(set(nodesout))
    return templates, nodesout
예제 #26
0
def brightness(stations,
               nodes,
               lags,
               stream,
               threshold,
               thresh_type,
               template_length,
               template_saveloc,
               coherence_thresh,
               coherence_stations=['all'],
               coherence_clip=False,
               gap=2.0,
               clip_level=100,
               instance=0,
               pre_pick=0.2,
               plotvar=False,
               plotsave=True,
               cores=1,
               debug=0,
               mem_issue=False):
    """
    Calculate the brightness function for a single day.

    Written to calculate the brightness function for a single day of data,
    using moveouts from a 3D travel-time grid.

    .. Note::
        Data in stream must be all of the same length and have the same
        sampling rates, see :func:`eqcorrscan.utils.pre_processing.dayproc`

    :type stations: list
    :param stations:
        List of station names from in the form where stations[i] refers to
        nodes[i][:] and lags[i][:]
    :type nodes: list
    :param nodes:
        List of node points where nodes[i] refers to stations[i] and
        nodes[:][:][0] is latitude in degrees, nodes[:][:][1] is longitude in
        degrees, nodes[:][:][2] is depth in km.
    :type lags: numpy.ndarray
    :param lags:
        Array of arrays where lags[i][:] refers to stations[i]. lags[i][j]
        should be the delay to the nodes[i][j] for stations[i] in seconds.
    :type stream: obspy.core.stream.Stream
    :param stream: Data through which to look for detections.
    :type threshold: float
    :param threshold:
        Threshold value for detection of template within the brightness
        function.
    :type thresh_type: str
    :param thresh_type:
        Either MAD or abs where MAD is the Median Absolute Deviation and abs
        is an absolute brightness.
    :type template_length: float
    :param template_length: Length of template to extract in seconds
    :type template_saveloc: str
    :param template_saveloc: Path of where to save the templates.
    :type coherence_thresh: tuple
    :param coherence_thresh:
            Threshold for removing incoherent peaks in the network response,
            those below this will not be used as templates. Must be in the
            form of (a,b) where the coherence is given by: :math:`a-kchan/b`
            where kchan is the number of channels used to compute the
            coherence.
    :type coherence_stations: list
    :param coherence_stations:
        List of stations to use in the coherence thresholding - defaults to
        `all` which uses all the stations.
    :type coherence_clip: tuple
    :param coherence_clip:
        Start and end in seconds of data to window around, defaults to False,
        which uses all the data given.
    :type gap: float
    :param gap: Minimum inter-event time in seconds for detections.
    :type clip_level: float
    :param clip_level:
        Multiplier applied to the mean deviation of the energy as an upper
        limit, used to remove spikes (earthquakes, lightning, electrical
        spikes) from the energy stack.
    :type instance: int
    :param instance:
        Optional, used for tracking when using a distributed computing system.
    :type pre_pick: float
    :param pre_pick: Seconds before the detection time to include in template
    :type plotvar: bool
    :param plotvar: Turn plotting on or off
    :type plotsave: bool
    :param plotsave:
        Save or show plots, if `False` will try and show the plots on screen -
        as this is designed for bulk use this is set to `True` to save any
        plots rather than show them if you create them - changes the backend
        of matplotlib, so if is set to `False` you will see NO PLOTS!
    :type cores: int
    :param cores: Number of cores to use, defaults to 1.
    :type debug: int
    :param debug: Debug level from 0-5, higher is more output.
    :type mem_issue: bool
    :param mem_issue:
        Set to True to write temporary variables to disk rather than store in
        memory, slow.

    :return: list of templates as :class:`obspy.core.stream.Stream` objects
    :rtype: list
    """
    if plotsave:
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
        plt.ioff()
    from eqcorrscan.utils import plotting
    from eqcorrscan.utils.debug_log import debug_print
    # Check that we actually have the correct stations
    realstations = []
    for station in stations:
        st = stream.select(station=station)
        if st:
            realstations += station
    del st
    stream_copy = stream.copy()
    # Force convert to int16
    for tr in stream_copy:
        # int16 max range is +/- 32767
        if max(abs(tr.data)) > 32767:
            tr.data = 32767 * (tr.data / max(abs(tr.data)))
            # Make sure that the data aren't clipped it they are high gain
            # scale the data
        tr.data = tr.data.astype(np.int16)
    # The internal _node_loop converts energy to int16 too to conserve memory,
    # to do this it forces the maximum of a single energy trace to be 500 and
    # normalises to this level - this only works for fewer than 65 channels of
    # data
    if len(stream_copy) > 130:
        raise BrightnessError(
            'Too many streams, either re-code and cope with either more memory'
            ' usage, or less precision, or reduce data volume')
    # Loop through each node in the input
    # Linear run
    print('Computing the energy stacks')
    # Parallel run
    num_cores = cores
    if num_cores > len(nodes):
        num_cores = len(nodes)
    if num_cores > cpu_count():
        num_cores = cpu_count()
    if mem_issue and not os.path.isdir('tmp' + str(instance)):
        os.makedirs('tmp' + str(instance))
    pool = Pool(processes=num_cores)
    results = [
        pool.apply_async(
            _node_loop, (stations, ), {
                'lags': lags[:, i],
                'stream': stream,
                'i': i,
                'clip_level': clip_level,
                'mem_issue': mem_issue,
                'instance': instance
            }) for i in range(len(nodes))
    ]
    pool.close()
    if not mem_issue:
        print('Computing the cumulative network response from memory')
        energy = [p.get() for p in results]
        pool.join()
        energy.sort(key=lambda tup: tup[0])
        energy = [node[1] for node in energy]
        energy = np.concatenate(energy, axis=0)
        print(energy.shape)
    else:
        pool.join()
        del results
    # Now compute the cumulative network response and then detect possible
    # events
    if not mem_issue:
        print(energy.shape)
        indices = np.argmax(energy, axis=0)  # Indices of maximum energy
        print(indices.shape)
        cum_net_resp = np.array([np.nan] * len(indices))
        cum_net_resp[0] = energy[indices[0]][0]
        peak_nodes = [nodes[indices[0]]]
        for i in range(1, len(indices)):
            cum_net_resp[i] = energy[indices[i]][i]
            peak_nodes.append(nodes[indices[i]])
        del energy, indices
    else:
        print('Reading the temp files and computing network response')
        node_splits = int(len(nodes) // num_cores)
        print(node_splits)
        indices = []
        for i in range(num_cores):
            indices.append(
                list(np.arange(node_splits * i, node_splits * (i + 1))))
        indices[-1] += list(np.arange(node_splits * (i + 1), len(nodes)))
        # results = [_cum_net_resp(node_lis=indices[i], instance=instance)
        #            for i in range(num_cores)]
        pool = Pool(processes=num_cores)
        results = [
            pool.apply_async(_cum_net_resp, args=(indices[i], instance))
            for i in range(num_cores)
        ]
        pool.close()
        results = [p.get() for p in results]
        pool.join()
        responses = [result[0] for result in results]
        print(np.shape(responses))
        node_indices = [result[1] for result in results]
        cum_net_resp = np.array(responses)
        indices = np.argmax(cum_net_resp, axis=0)
        print(indices.shape)
        print(cum_net_resp.shape)
        cum_net_resp = np.array(
            [cum_net_resp[indices[i]][i] for i in range(len(indices))])
        peak_nodes = [
            nodes[node_indices[indices[i]][i]] for i in range(len(indices))
        ]
        del indices, node_indices
    if plotvar:
        cum_net_trace = Stream(
            Trace(data=cum_net_resp,
                  header=Stats({
                      'station': 'NR',
                      'channel': '',
                      'network': 'Z',
                      'location': '',
                      'starttime': stream[0].stats.starttime,
                      'sampling_rate': stream[0].stats.sampling_rate
                  })))
        cum_net_trace += stream.select(channel='*N')
        cum_net_trace += stream.select(channel='*1')
        cum_net_trace.sort(['network', 'station', 'channel'])

    # Find detection within this network response
    print('Finding detections in the cumulative network response')
    detections = _find_detections(cum_net_resp, peak_nodes, threshold,
                                  thresh_type, stream[0].stats.sampling_rate,
                                  realstations, gap)
    del cum_net_resp
    templates = []
    nodesout = []
    good_detections = []
    if detections:
        print('Converting detections into templates')
        # Generate a catalog of detections
        # detections_cat = Catalog()
        for j, detection in enumerate(detections):
            debug_print(
                'Converting for detection %i of %i' % (j, len(detections)), 3,
                debug)
            # Create an event for each detection
            event = Event()
            # Set up some header info for the event
            event.event_descriptions.append(EventDescription())
            event.event_descriptions[0].text = 'Brightness detection'
            event.creation_info = CreationInfo(agency_id='EQcorrscan')
            copy_of_stream = deepcopy(stream_copy)
            # Convert detections to obspy.core.event type -
            # name of detection template is the node.
            node = (detection.template_name.split('_')[0],
                    detection.template_name.split('_')[1],
                    detection.template_name.split('_')[2])
            # Look up node in nodes and find the associated lags
            index = nodes.index(
                (float(node[0]), float(node[1]), float(node[2])))
            detect_lags = lags[:, index]
            ksta = Comment(text='Number of stations=' + str(len(detect_lags)))
            event.origins.append(Origin())
            event.origins[0].comments.append(ksta)
            event.origins[0].time = copy_of_stream[0].stats.starttime +\
                detect_lags[0] + detection.detect_time
            event.origins[0].latitude = node[0]
            event.origins[0].longitude = node[1]
            event.origins[0].depth = node[2]
            for i, detect_lag in enumerate(detect_lags):
                station = stations[i]
                st = copy_of_stream.select(station=station)
                if len(st) != 0:
                    for tr in st:
                        _waveform_id = WaveformStreamID(
                            station_code=tr.stats.station,
                            channel_code=tr.stats.channel,
                            network_code=tr.stats.network)
                        event.picks.append(
                            Pick(waveform_id=_waveform_id,
                                 time=tr.stats.starttime + detect_lag +
                                 detection.detect_time + pre_pick,
                                 onset='emergent',
                                 evalutation_mode='automatic'))
            debug_print('Generating template for detection: %i' % j, 0, debug)
            template = _template_gen(picks=event.picks,
                                     st=copy_of_stream,
                                     length=template_length,
                                     swin='all')
            template_name = template_saveloc + '/' +\
                str(template[0].stats.starttime) + '.ms'
            # In the interests of RAM conservation we write then read
            # Check coherency here!
            temp_coher, kchan = coherence(template, coherence_stations,
                                          coherence_clip)
            coh_thresh = float(coherence_thresh[0]) - kchan / \
                float(coherence_thresh[1])
            coherent = False
            if temp_coher > coh_thresh:
                template.write(template_name, format="MSEED")
                print('Written template as: ' + template_name)
                print('---------------------------------coherence LEVEL: ' +
                      str(temp_coher))
                coherent = True
                debug_print(
                    'Template was incoherent, coherence level: ' +
                    str(temp_coher), 0, debug)
                coherent = False
            del copy_of_stream, tr, template
            if coherent:
                templates.append(obsread(template_name))
                nodesout += [node]
                good_detections.append(detection)
            debug_print('No template for you', 0, debug)
            # detections_cat += event
    if plotvar:
        good_detections = [(cum_net_trace[-1].stats.starttime +
                            detection.detect_time).datetime
                           for detection in good_detections]
        if not plotsave:
            plotting.NR_plot(cum_net_trace[0:-1],
                             Stream(cum_net_trace[-1]),
                             detections=good_detections,
                             size=(18.5, 10),
                             title='Network response')
            # cum_net_trace.plot(size=(800,600), equal_scale=False)
        else:
            savefile = 'plots/' +\
                cum_net_trace[0].stats.starttime.datetime.strftime('%Y%m%d') +\
                '_NR_timeseries.pdf'
            plotting.NR_plot(cum_net_trace[0:-1],
                             Stream(cum_net_trace[-1]),
                             detections=good_detections,
                             size=(18.5, 10),
                             save=True,
                             savefile=savefile,
                             title='Network response')
    nodesout = list(set(nodesout))
    return templates, nodesout