def test_warn_zeros(self): st = self.st.copy() template = template_gen(self.picks, st.copy(), 10) self.assertTrue('LABE' in [tr.stats.station for tr in template]) st.select(station='LABE', channel='SN')[0].data = np.zeros(10000) template = template_gen(self.picks, st, 10) self.assertFalse('LABE' in [tr.stats.station for tr in template])
def test_not_delayed(self): """Test the method of template_gen without applying delays to channels.""" client = Client("GEONET") cat = client.get_events(minlatitude=-40.98, maxlatitude=-40.85, minlongitude=175.4, maxlongitude=175.5, starttime=UTCDateTime(2016, 5, 1), endtime=UTCDateTime(2016, 5, 2)) cat = filter_picks(catalog=cat, top_n_picks=5) template = template_gen(method="from_client", catalog=cat, client_id='GEONET', lowcut=None, highcut=None, samp_rate=100.0, filt_order=4, length=10.0, prepick=0.5, swin='all', process_len=3600, plot=False, delayed=False)[0] for tr in template: tr.stats.starttime.precision = 6 starttime = template[0].stats.starttime length = template[0].stats.npts self.assertEqual(len(template), 5) for tr in template: self.assertTrue( abs((tr.stats.starttime - starttime)) <= tr.stats.delta) self.assertEqual(tr.stats.npts, length) template = template_gen(method="from_client", catalog=cat, client_id='GEONET', lowcut=None, highcut=None, samp_rate=100.0, filt_order=4, length=10.0, prepick=0.5, swin='P_all', process_len=3600, plot=False, delayed=False)[0] for tr in template: tr.stats.starttime.precision = 6 starttime = template[0].stats.starttime length = template[0].stats.npts self.assertEqual(len(template), 15) for tr in template: self.assertTrue( abs((tr.stats.starttime - starttime)) <= tr.stats.delta) self.assertEqual(tr.stats.npts, length)
def test_sac_template_gen(self): """Test template generation.""" samp_rate = 20 length = 8 for event in ['2014p611252', 'No_head']: test_files = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'test_data', 'SAC', event, '*') # Test with various input types filelist = glob.glob(test_files) streamlist = [read(f) for f in glob.glob(test_files)] stream = read(test_files) for sac_files in [filelist, streamlist, stream]: templates = template_gen(method="from_sac", sac_files=sac_files, lowcut=2.0, highcut=8.0, samp_rate=samp_rate, filt_order=4, length=length, swin='all', prepick=0.1, plot=False) self.assertEqual(len(templates), 1) template = templates[0] self.assertEqual(len(template), len(sactoevent(stream).picks)) for tr in template: self.assertEqual(len(tr.data), length * samp_rate)
def test_save_progress(self): """ Test template creation with progress saving """ client = Client('GEONET') catalog = client.get_events(starttime=UTCDateTime(2016, 1, 4, 0, 50), endtime=UTCDateTime(2016, 1, 4, 1, 20)) # Gets a catalog of 2 events separated by 127s # Need a bigger gap to allow moveouts catalog[0].origins[0].time -= 600 for pick in catalog[0].picks: pick.time -= 600 catalog = filter_picks(catalog=catalog, top_n_picks=5) templates = template_gen(method="from_client", catalog=catalog, client_id="GEONET", lowcut=2, highcut=5, samp_rate=20, filt_order=4, prepick=0.4, process_len=600, swin="P", save_progress=True, length=2) assert (os.path.isdir("eqcorrscan_temporary_templates")) saved_templates = [ read(f) for f in sorted(glob.glob("eqcorrscan_temporary_templates/*.ms")) ] # Writing to miniseed adds miniseed stats dict for saved_template, template in zip(saved_templates, templates): for saved_tr in saved_template: tr = template.select(id=saved_tr.id)[0] assert (np.allclose(saved_tr.data, tr.data, atol=0.01)) shutil.rmtree("eqcorrscan_temporary_templates")
def test_bad_client(self): """Ensure passing a non-client raises.""" client = _StreamTestClient() cat = client.get_default_catalog() with self.assertRaises(NotImplementedError): template_gen('from_client', client_id=cat, catalog=cat, highcut=None, lowcut=None, filt_order=4, samp_rate=100, prepick=0.1, length=10, process_len=20, data_pad=5)
def test_download_various_methods(self): """ Will download data from server and store in various databases, then create templates using the various methods. """ client = Client('GEONET') # get the events catalog = client.get_events(eventid='2016p008194') # Select 3 channels to use and download sta_chans = [(pick.waveform_id.station_code, pick.waveform_id.channel_code) for pick in catalog[0].picks[0:3]] t1 = UTCDateTime(catalog[0].origins[0].time.date) t2 = t1 + 86400 bulk = [('NZ', sta_chan[0], '*', sta_chan[1], t1, t2) for sta_chan in sta_chans] continuous_st = client.get_waveforms_bulk(bulk) continuous_st.merge(fill_value=0) # Test multi_template_gen kwargs = { "process": False, "lowcut": None, "highcut": None, "filt_order": None, "swin": "all", "prepick": 0.05, "all_horiz": False, "delayed": True, "plot": False, "return_event": False, "min_snr": None, "samp_rate": continuous_st[0].stats.sampling_rate } templates = template_gen(method="from_meta_file", meta_file=catalog, st=continuous_st, length=3, **kwargs) self.assertEqual(len(templates), 1) # Test without an event templates = template_gen(method="from_meta_file", meta_file=Catalog(), st=continuous_st, length=3, **kwargs) self.assertEqual(len(templates), 0)
def mktemplates( network_code='GEONET', plot=True, publicIDs=['2016p008122', '2016p008353', '2016p008155', '2016p008194']): """Functional wrapper to make templates""" client = Client(network_code) # We want to download a few events from an earthquake sequence, these are # identified by publiID numbers, given as arguments catalog = Catalog() for publicID in publicIDs: try: catalog += client.get_events(eventid=publicID, includearrivals=True) except TypeError: # Cope with some FDSN services not implementing includearrivals catalog += client.get_events(eventid=publicID) # Lets plot the catalog to see what we have if plot: catalog.plot(projection='local', resolution='h') # We don't need all the picks, lets take the information from the # five most used stations - note that this is done to reduce computational # costs. catalog = filter_picks(catalog, top_n_picks=5) # We only want the P picks in this example, but you can use others or all # picks if you want. for event in catalog: for pick in event.picks: if pick.phase_hint == 'S': event.picks.remove(pick) # Now we can generate the templates templates = template_gen.template_gen(method='from_client', catalog=catalog, client_id=network_code, lowcut=2.0, highcut=9.0, samp_rate=20.0, filt_order=4, length=3.0, prepick=0.15, swin='all', process_len=3600, plot=plot) # We now have a series of templates! Using Obspy's Stream.write() method we # can save these to disk for later use. We will do that now for use in the # following tutorials. for i, template in enumerate(templates): template.write('tutorial_template_' + str(i) + '.ms', format='MSEED') # Note that this will warn you about data types. As we don't care # at the moment, whatever obspy chooses is fine. return
def test_seishub(self): """Test the seishub method, use obspy default seishub client.""" import sys if sys.version_info.major == 2: from future.backports.urllib.request import URLError else: from urllib.request import URLError t = UTCDateTime(2009, 9, 3) test_cat = Catalog() test_cat.append(Event()) test_cat[0].origins.append(Origin()) test_cat[0].origins[0].time = t test_cat[0].origins[0].latitude = 45 test_cat[0].origins[0].longitude = 45 test_cat[0].origins[0].depth = 5000 test_cat[0].picks.append( Pick(waveform_id=WaveformStreamID(station_code='MANZ', channel_code='EHZ', network_code='BW'), phase_hint='PG', time=t + 2000)) test_cat[0].picks.append( Pick(waveform_id=WaveformStreamID(station_code='MANZ', channel_code='EHN', network_code='BW'), phase_hint='SG', time=t + 2005)) test_cat[0].picks.append( Pick(waveform_id=WaveformStreamID(station_code='MANZ', channel_code='EHE', network_code='BW'), phase_hint='SG', time=t + 2005.5)) test_url = "http://teide.geophysik.uni-muenchen.de:8080" if sys.version_info.major == 3: try: template = template_gen(method="from_seishub", catalog=test_cat, url=test_url, lowcut=1.0, highcut=5.0, samp_rate=20, filt_order=4, length=3, prepick=0.5, swin='all', process_len=300) except URLError: pass else: pass if 'template' in locals(): self.assertEqual(len(template), 3)
def test_extract_from_stack(self): length = 3 stack = self.st.copy() template = template_gen(self.picks, self.st.copy(), 2) extracted = extract_from_stack(stack, template, length=length, pre_pick=0.3, pre_pad=45) self.assertEqual(len(template), len(extracted)) for tr in extracted: self.assertEqual(tr.stats.endtime - tr.stats.starttime, length)
def mktemplates(network_code='GEONET', publicIDs=['2016p008122', '2016p008353', '2016p008155', '2016p008194'], plot=True): """Functional wrapper to make templates""" # We want to download some QuakeML files from the New Zealand GeoNet # network, GeoNet currently doesn't support FDSN event queries, so we # have to work around to download quakeml from their quakeml.geonet site. client = Client(network_code) # We want to download a few events from an earthquake sequence, these are # identified by publiID numbers, given as arguments catalog = Catalog() for publicID in publicIDs: if network_code == 'GEONET': data_stream = client._download( 'http://quakeml.geonet.org.nz/quakeml/1.2/' + publicID) data_stream.seek(0, 0) catalog += read_events(data_stream, format="quakeml") data_stream.close() else: catalog += client.get_events( eventid=publicID, includearrivals=True) # Lets plot the catalog to see what we have if plot: catalog.plot(projection='local', resolution='h') # We don't need all the picks, lets take the information from the # five most used stations - note that this is done to reduce computational # costs. catalog = filter_picks(catalog, top_n_picks=5) # We only want the P picks in this example, but you can use others or all # picks if you want. for event in catalog: for pick in event.picks: if pick.phase_hint == 'S': event.picks.remove(pick) # Now we can generate the templates templates = template_gen.template_gen( method='from_client', catalog=catalog, client_id=network_code, lowcut=2.0, highcut=9.0, samp_rate=20.0, filt_order=4, length=3.0, prepick=0.15, swin='all', process_len=3600, debug=0, plot=plot) # We now have a series of templates! Using Obspy's Stream.write() method we # can save these to disk for later use. We will do that now for use in the # following tutorials. for i, template in enumerate(templates): template.write('tutorial_template_' + str(i) + '.ms', format='MSEED') # Note that this will warn you about data types. As we don't care # at the moment, whatever obspy chooses is fine. return
def test_day_long_processing(self): templates = template_gen(method='from_meta_file', meta_file=self.cat, st=self.st, lowcut=2.0, highcut=9.0, samp_rate=20.0, filt_order=3, length=2, prepick=0.1, swin='P', all_horiz=True) self.assertEqual(len(templates), 1) self.assertEqual(len(templates[0]), 3)
def test_extract_from_stack_and_process(self): length = 3 stack = self.st.copy() template = template_gen(self.picks, self.st.copy(), 2) extracted = extract_from_stack(stack, template, length=length, pre_pick=0.3, pre_pad=45, pre_processed=False, samp_rate=20, lowcut=2, highcut=8) self.assertEqual(len(template), len(extracted)) for tr in extracted: self.assertEqual(tr.stats.endtime - tr.stats.starttime, length)
def test_triggered_data(self): client = Client("GEONET") catalog = client.get_events(eventid="1481730") templates = template_gen("from_client", lowcut=2., highcut=15., samp_rate=40., swin="all", filt_order=4, prepick=0.2, catalog=catalog, length=3.0, client_id="GEONET", all_horiz=True, process_len=600, min_snr=5., skip_short_chans=True) self.assertEqual(len(templates), 0)
def test_from_client(self): """Test for using a waveform client not related to obspy's Clients.""" client = _StreamTestClient() cat = client.get_default_catalog() temps = template_gen('from_client', client_id=client, catalog=cat, highcut=None, lowcut=None, filt_order=4, samp_rate=100, prepick=0.1, length=10, process_len=20, data_pad=5) self.assertEqual(len(temps), 1) # there should be one template stream self.assertEqual(len(temps[0]), 1) # with one trace self.assertGreater(len(temps[0][0].data), 1) # with some data
def test_missing_waveform_id(self): testing_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'test_data') quakeml = os.path.join(testing_path, '20130901T041115_missingwavid.xml') st = read( os.path.join(testing_path, 'WAV', 'TEST_', '2013-09-01-0410-35.DFDPC_024_00')) templates = template_gen(method="from_meta_file", meta_file=quakeml, st=st, lowcut=2.0, highcut=9.0, samp_rate=20.0, filt_order=3, length=2, prepick=0.1, swin='S') self.assertEqual(len(templates), 1)
def test_undefined_phase_type(self): with self.assertRaises(IOError): template_gen(picks=self.picks, st=self.st.copy(), length=2, swin='bob')
def test_missing_data(self): picks = copy.deepcopy(self.picks) picks.append(picks[-1]) picks[-1].waveform_id.station_code = 'DUMMY' template = template_gen(picks, self.st.copy(), 10) self.assertFalse('DUMMY' in [tr.stats.station for tr in template])
def test_no_matched_picks(self): picks = [copy.deepcopy(self.picks[0])] picks[0].waveform_id.station_code = 'DUMMY' template = template_gen(picks, self.st.copy(), 10) self.assertFalse(template)
def test_all_phase_methods(self): sfile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'test_data', 'REA', 'TEST_', '01-0411-15L.S201309') catalog = read_events(sfile) p_stations = list( set([ pick.waveform_id.station_code for pick in catalog[0].picks if pick.phase_hint == 'P' ])) s_stations = list( set([ pick.waveform_id.station_code for pick in catalog[0].picks if pick.phase_hint == 'S' ])) st = read( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'test_data', 'WAV', 'TEST_', '2013-09-01-0410-35.DFDPC_024_00')) templates = template_gen(method="from_meta_file", meta_file=sfile, st=st, lowcut=2, highcut=20, samp_rate=100, filt_order=4, length=6, prepick=0.2, swin='P_all') self.assertEqual(len(templates), 1) self.assertEqual(len(templates[0]), len(p_stations) * 3) for tr in templates[0]: pick = [ p for p in catalog[0].picks if p.waveform_id.station_code == tr.stats.station and p.phase_hint.upper() == 'P' ][0] print(tr) print(pick) self.assertLess(abs(tr.stats.starttime - (pick.time - 0.2)), tr.stats.delta) templates = template_gen(method="from_meta_file", meta_file=sfile, st=st, lowcut=2, highcut=20, samp_rate=100, filt_order=4, length=6, prepick=0.2, swin='S_all') self.assertEqual(len(templates), 1) self.assertEqual(len(templates[0]), len(s_stations) * 3) for tr in templates[0]: pick = [ p for p in catalog[0].picks if p.waveform_id.station_code == tr.stats.station and p.phase_hint.upper() == 'S' ][0] print(tr) print(pick) self.assertLess(abs(tr.stats.starttime - (pick.time - 0.2)), tr.stats.delta)
def mseed_2_templates(wav_dirs, cat, outdir, length, prepick, highcut=None, lowcut=None, f_order=None, samp_rate=None, min_snr=2., start=None, end=None, miniseed=True, asdf_file=False, debug=1): """ Function to generate individual mseed files for each event in a catalog from a pyasdf file or continuous data. :param asdf_file: ASDF file with waveforms and stations :param cat: path to xml of Catalog of events for which we'll create templates :param outdir: output directory for miniseed files :param length: length of templates in seconds :param prepick: prepick time for waveform trimming :param highcut: Filter highcut (if desired) :param lowcut: Filter lowcut (if desired) :param f_order: Filter order :param samp_rate: Sampling rate for the templates :param start: start date as %Y/%m/%d if desired :param end: same as above. Defaults to full length of catalog. :return: """ # Establish date range for template creation cat.events.sort(key=lambda x: x.origins[-1].time) if start: cat_start = datetime.datetime.strptime(start, '%d/%m/%Y') cat_end = datetime.datetime.strptime(end, '%d/%m/%Y') else: cat_start = cat[0].origins[-1].time.date cat_end = cat[-1].origins[-1].time.date for date in date_generator(cat_start, cat_end): dto = UTCDateTime(date) print('Processing templates for: %s' % str(dto)) q_start = dto - 10 q_end = dto + 86410 # Establish which events are in this day sch_str_start = 'time >= %s' % str(dto) sch_str_end = 'time <= %s' % str(dto + 86400) tmp_cat = cat.filter(sch_str_start, sch_str_end) if len(tmp_cat) == 0: print('No events on: %s' % str(dto)) continue # Which stachans we got? stachans = {pk.waveform_id.station_code: [] for ev in tmp_cat for pk in ev.picks} for ev in tmp_cat: for pk in ev.picks: chan_code = pk.waveform_id.channel_code if chan_code not in stachans[pk.waveform_id.station_code]: stachans[pk.waveform_id.station_code].append(chan_code) wav_read_start = timer() # Be sure to go +/- 10 sec to account for GeoNet shit timing if asdf_file: with pyasdf.ASDFDataSet(asdf_file) as ds: st = Stream() for sta, chans in iter(stachans.items()): for station in ds.ifilter(ds.q.station == sta, ds.q.channel == chans, ds.q.starttime >= q_start, ds.q.endtime <= q_end): st += station.raw_recording elif miniseed: wav_ds = ['%s%d' % (d, dto.year) for d in wav_dirs] st = grab_day_wavs(wav_ds, dto, stachans) wav_read_stop = timer() print('Reading waveforms took %.3f seconds' % (wav_read_stop - wav_read_start)) print('Looping through stachans to merge/resamp') stachans = [(tr.stats.station, tr.stats.channel) for tr in st] for stachan in list(set(stachans)): tmp_st = st.select(station=stachan[0], channel=stachan[1]) if len(tmp_st) > 1 and len(set([tr.stats.sampling_rate for tr in tmp_st])) > 1: print('Traces from %s.%s have differing samp rates' % (stachan[0], stachan[1])) for tr in tmp_st: st.remove(tr) tmp_st.resample(sampling_rate=samp_rate) st += tmp_st st.merge(fill_value='interpolate') resamp_stop = timer() print('Resample/merge took %s secs' % str(resamp_stop - wav_read_stop)) print('Preprocessing...') # Process the stream try: st1 = pre_processing.dayproc(st, lowcut=lowcut, highcut=highcut, filt_order=f_order, samp_rate=samp_rate, starttime=dto, debug=debug, ignore_length=True, num_cores=4) except NotImplementedError or Exception as e: print('Found error in dayproc, noting date and continuing') print(e) with open('%s/dayproc_errors.txt' % outdir, mode='a') as fo: fo.write('%s\n%s\n' % (str(date), e)) continue print('Feeding stream to template_gen...') for event in tmp_cat: print('Copying stream to keep away from the trim...') trim_st = copy.deepcopy(st1) ev_name = str(event.resource_id).split('/')[-1] pk_stachans = ['%s.%s' % (pk.waveform_id.station_code, pk.waveform_id.channel_code) for pk in event.picks] # Run check to ensure that there is only one pick for each channel dups = [pk for pk, count in collections.Counter(pk_stachans).items() if count > 1] if len(dups) > 0: print('Event %s still has dup picks. Skipping' % ev_name) continue template = template_gen(event.picks, trim_st, length=length, prepick=prepick, min_snr=min_snr) if len([tr for tr in template if tr.stats.channel[-1] == 'Z']) < 6: print('Skipping template with fewer than 6 Z-comp traces') continue # temp_list.append(template) print('Writing event %s to file...' % ev_name) template.write('%s/%s.mseed' % (outdir, ev_name), format="MSEED") del trim_st del tmp_cat, st1, st
def brightness(stations, nodes, lags, stream, threshold, thresh_type, template_length, template_saveloc, coherence_thresh, coherence_stations=['all'], coherence_clip=False, gap=2.0, clip_level=100, instance=0, pre_pick=0.2, plotvar=False, plotsave=True, cores=1, debug=0, mem_issue=False): """ Calculate the brightness function for a single day. Written to calculate the brightness function for a single day of data, using moveouts from a 3D travel-time grid. .. Note:: Data in stream must be all of the same length and have the same sampling rates, see :func:`eqcorrscan.utils.pre_processing.dayproc` :type stations: list :param stations: List of station names from in the form where stations[i] refers to nodes[i][:] and lags[i][:] :type nodes: list :param nodes: List of node points where nodes[i] refers to stations[i] and nodes[:][:][0] is latitude in degrees, nodes[:][:][1] is longitude in degrees, nodes[:][:][2] is depth in km. :type lags: numpy.ndarray :param lags: Array of arrays where lags[i][:] refers to stations[i]. lags[i][j] should be the delay to the nodes[i][j] for stations[i] in seconds. :type stream: obspy.core.stream.Stream :param stream: Data through which to look for detections. :type threshold: float :param threshold: Threshold value for detection of template within the brightness function. :type thresh_type: str :param thresh_type: Either MAD or abs where MAD is the Median Absolute Deviation and abs is an absolute brightness. :type template_length: float :param template_length: Length of template to extract in seconds :type template_saveloc: str :param template_saveloc: Path of where to save the templates. :type coherence_thresh: tuple :param coherence_thresh: Threshold for removing incoherent peaks in the network response, those below this will not be used as templates. Must be in the form of (a,b) where the coherence is given by: :math:`a-kchan/b` where kchan is the number of channels used to compute the coherence. :type coherence_stations: list :param coherence_stations: List of stations to use in the coherence thresholding - defaults to `all` which uses all the stations. :type coherence_clip: tuple :param coherence_clip: Start and end in seconds of data to window around, defaults to False, which uses all the data given. :type gap: float :param gap: Minimum inter-event time in seconds for detections. :type clip_level: float :param clip_level: Multiplier applied to the mean deviation of the energy as an upper limit, used to remove spikes (earthquakes, lightning, electrical spikes) from the energy stack. :type instance: int :param instance: Optional, used for tracking when using a distributed computing system. :type pre_pick: float :param pre_pick: Seconds before the detection time to include in template :type plotvar: bool :param plotvar: Turn plotting on or off :type plotsave: bool :param plotsave: Save or show plots, if `False` will try and show the plots on screen - as this is designed for bulk use this is set to `True` to save any plots rather than show them if you create them - changes the backend of matplotlib, so if is set to `False` you will see NO PLOTS! :type cores: int :param cores: Number of cores to use, defaults to 1. :type debug: int :param debug: Debug level from 0-5, higher is more output. :type mem_issue: bool :param mem_issue: Set to True to write temporary variables to disk rather than store in memory, slow. :return: list of templates as :class:`obspy.core.stream.Stream` objects :rtype: list """ if plotsave: import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt plt.ioff() from eqcorrscan.utils import plotting from eqcorrscan.utils.debug_log import debug_print # Check that we actually have the correct stations realstations = [] for station in stations: st = stream.select(station=station) if st: realstations += station del st stream_copy = stream.copy() # Force convert to int16 for tr in stream_copy: # int16 max range is +/- 32767 if max(abs(tr.data)) > 32767: tr.data = 32767 * (tr.data / max(abs(tr.data))) # Make sure that the data aren't clipped it they are high gain # scale the data tr.data = tr.data.astype(np.int16) # The internal _node_loop converts energy to int16 too to conserve memory, # to do this it forces the maximum of a single energy trace to be 500 and # normalises to this level - this only works for fewer than 65 channels of # data if len(stream_copy) > 130: raise BrightnessError( 'Too many streams, either re-code and cope with either more memory' ' usage, or less precision, or reduce data volume') # Loop through each node in the input # Linear run print('Computing the energy stacks') # Parallel run num_cores = cores if num_cores > len(nodes): num_cores = len(nodes) if num_cores > cpu_count(): num_cores = cpu_count() if mem_issue and not os.path.isdir('tmp' + str(instance)): os.makedirs('tmp' + str(instance)) pool = Pool(processes=num_cores) results = [ pool.apply_async( _node_loop, (stations, ), { 'lags': lags[:, i], 'stream': stream, 'i': i, 'clip_level': clip_level, 'mem_issue': mem_issue, 'instance': instance }) for i in range(len(nodes)) ] pool.close() if not mem_issue: print('Computing the cumulative network response from memory') energy = [p.get() for p in results] pool.join() energy.sort(key=lambda tup: tup[0]) energy = [node[1] for node in energy] energy = np.concatenate(energy, axis=0) print(energy.shape) else: pool.join() del results # Now compute the cumulative network response and then detect possible # events if not mem_issue: print(energy.shape) indices = np.argmax(energy, axis=0) # Indices of maximum energy print(indices.shape) cum_net_resp = np.array([np.nan] * len(indices)) cum_net_resp[0] = energy[indices[0]][0] peak_nodes = [nodes[indices[0]]] for i in range(1, len(indices)): cum_net_resp[i] = energy[indices[i]][i] peak_nodes.append(nodes[indices[i]]) del energy, indices else: print('Reading the temp files and computing network response') node_splits = int(len(nodes) // num_cores) print(node_splits) indices = [] for i in range(num_cores): indices.append( list(np.arange(node_splits * i, node_splits * (i + 1)))) indices[-1] += list(np.arange(node_splits * (i + 1), len(nodes))) # results = [_cum_net_resp(node_lis=indices[i], instance=instance) # for i in range(num_cores)] pool = Pool(processes=num_cores) results = [ pool.apply_async(_cum_net_resp, args=(indices[i], instance)) for i in range(num_cores) ] pool.close() results = [p.get() for p in results] pool.join() responses = [result[0] for result in results] print(np.shape(responses)) node_indices = [result[1] for result in results] cum_net_resp = np.array(responses) indices = np.argmax(cum_net_resp, axis=0) print(indices.shape) print(cum_net_resp.shape) cum_net_resp = np.array( [cum_net_resp[indices[i]][i] for i in range(len(indices))]) peak_nodes = [ nodes[node_indices[indices[i]][i]] for i in range(len(indices)) ] del indices, node_indices if plotvar: cum_net_trace = Stream( Trace(data=cum_net_resp, header=Stats({ 'station': 'NR', 'channel': '', 'network': 'Z', 'location': '', 'starttime': stream[0].stats.starttime, 'sampling_rate': stream[0].stats.sampling_rate }))) cum_net_trace += stream.select(channel='*N') cum_net_trace += stream.select(channel='*1') cum_net_trace.sort(['network', 'station', 'channel']) # Find detection within this network response print('Finding detections in the cumulative network response') detections = _find_detections(cum_net_resp, peak_nodes, threshold, thresh_type, stream[0].stats.sampling_rate, realstations, gap) del cum_net_resp templates = [] nodesout = [] good_detections = [] if detections: print('Converting detections into templates') # Generate a catalog of detections # detections_cat = Catalog() for j, detection in enumerate(detections): debug_print( 'Converting for detection %i of %i' % (j, len(detections)), 3, debug) # Create an event for each detection event = Event() # Set up some header info for the event event.event_descriptions.append(EventDescription()) event.event_descriptions[0].text = 'Brightness detection' event.creation_info = CreationInfo(agency_id='EQcorrscan') copy_of_stream = deepcopy(stream_copy) # Convert detections to obspy.core.event type - # name of detection template is the node. node = (detection.template_name.split('_')[0], detection.template_name.split('_')[1], detection.template_name.split('_')[2]) # Look up node in nodes and find the associated lags index = nodes.index( (float(node[0]), float(node[1]), float(node[2]))) detect_lags = lags[:, index] ksta = Comment(text='Number of stations=' + str(len(detect_lags))) event.origins.append(Origin()) event.origins[0].comments.append(ksta) event.origins[0].time = copy_of_stream[0].stats.starttime +\ detect_lags[0] + detection.detect_time event.origins[0].latitude = node[0] event.origins[0].longitude = node[1] event.origins[0].depth = node[2] for i, detect_lag in enumerate(detect_lags): station = stations[i] st = copy_of_stream.select(station=station) if len(st) != 0: for tr in st: _waveform_id = WaveformStreamID( station_code=tr.stats.station, channel_code=tr.stats.channel, network_code=tr.stats.network) event.picks.append( Pick(waveform_id=_waveform_id, time=tr.stats.starttime + detect_lag + detection.detect_time + pre_pick, onset='emergent', evalutation_mode='automatic')) debug_print('Generating template for detection: %i' % j, 0, debug) template = template_gen(picks=event.picks, st=copy_of_stream, length=template_length, swin='all') template_name = template_saveloc + '/' +\ str(template[0].stats.starttime) + '.ms' # In the interests of RAM conservation we write then read # Check coherency here! temp_coher, kchan = coherence(template, coherence_stations, coherence_clip) coh_thresh = float(coherence_thresh[0]) - kchan / \ float(coherence_thresh[1]) coherent = False if temp_coher > coh_thresh: template.write(template_name, format="MSEED") print('Written template as: ' + template_name) print('---------------------------------coherence LEVEL: ' + str(temp_coher)) coherent = True debug_print( 'Template was incoherent, coherence level: ' + str(temp_coher), 0, debug) coherent = False del copy_of_stream, tr, template if coherent: templates.append(obsread(template_name)) nodesout += [node] good_detections.append(detection) debug_print('No template for you', 0, debug) # detections_cat += event if plotvar: good_detections = [(cum_net_trace[-1].stats.starttime + detection.detect_time).datetime for detection in good_detections] if not plotsave: plotting.NR_plot(cum_net_trace[0:-1], Stream(cum_net_trace[-1]), detections=good_detections, size=(18.5, 10), title='Network response') # cum_net_trace.plot(size=(800,600), equal_scale=False) else: savefile = 'plots/' +\ cum_net_trace[0].stats.starttime.datetime.strftime('%Y%m%d') +\ '_NR_timeseries.pdf' plotting.NR_plot(cum_net_trace[0:-1], Stream(cum_net_trace[-1]), detections=good_detections, size=(18.5, 10), save=True, savefile=savefile, title='Network response') nodesout = list(set(nodesout)) return templates, nodesout
def construct(self, method, name, lowcut, highcut, samp_rate, filt_order, length, prepick, swin="all", process_len=86400, all_horiz=False, delayed=True, plot=False, plotdir=None, min_snr=None, parallel=False, num_cores=False, skip_short_chans=False, **kwargs): """ Construct a template using a given method. :param method: Method to make the template, the only available method is: `from_sac`. For all other methods (`from_seishub`, `from_client` and `from_meta_file`) use `Tribe.construct()`. :type method: str :type name: str :param name: Name for the template :type lowcut: float :param lowcut: Low cut (Hz), if set to None will not apply a lowcut :type highcut: float :param highcut: High cut (Hz), if set to None will not apply a highcut. :type samp_rate: float :param samp_rate: New sampling rate in Hz. :type filt_order: int :param filt_order: Filter level (number of corners). :type length: float :param length: Length of template waveform in seconds. :type prepick: float :param prepick: Pre-pick time in seconds :type swin: str :param swin: P, S, P_all, S_all or all, defaults to all: see note in :func:`eqcorrscan.core.template_gen.template_gen` :type process_len: int :param process_len: Length of data in seconds to download and process. :type all_horiz: bool :param all_horiz: To use both horizontal channels even if there is only a pick on one of them. Defaults to False. :type delayed: bool :param delayed: If True, each channel will begin relative to it's own pick-time, if set to False, each channel will begin at the same time. :type plot: bool :param plot: Plot templates or not. :type plotdir: str  :param plotdir: The path to save plots to. If `plotdir=None` (default) then the figure will be shown on screen. :type min_snr: float :param min_snr: Minimum signal-to-noise ratio for a channel to be included in the template, where signal-to-noise ratio is calculated as the ratio of the maximum amplitude in the template window to the rms amplitude in the whole window given. :type parallel: bool :param parallel: Whether to process data in parallel or not. :type num_cores: int :param num_cores: Number of cores to try and use, if False and parallel=True, will use either all your cores, or as many traces as in the data (whichever is smaller). :type skip_short_chans: bool :param skip_short_chans: Whether to ignore channels that have insufficient length data or not. Useful when the quality of data is not known, e.g. when downloading old, possibly triggered data from a datacentre .. note:: `method=from_sac` requires the following kwarg(s): :param list sac_files: osbpy.core.stream.Stream of sac waveforms, or list of paths to sac waveforms. .. note:: See `eqcorrscan.utils.sac_util.sactoevent` for details on how pick information is collected. .. rubric:: Example >>> # Get the path to the test data >>> import eqcorrscan >>> import os, glob >>> TEST_PATH = ( ... os.path.dirname(eqcorrscan.__file__) + '/tests/test_data') >>> sac_files = glob.glob(TEST_PATH + '/SAC/2014p611252/*') >>> template = Template().construct( ... method='from_sac', name='test', lowcut=2.0, highcut=8.0, ... samp_rate=20.0, filt_order=4, prepick=0.1, swin='all', ... length=2.0, sac_files=sac_files) >>> print(template) # doctest: +NORMALIZE_WHITESPACE Template test: 12 channels; lowcut: 2.0 Hz; highcut: 8.0 Hz; sampling rate 20.0 Hz; filter order: 4; process length: 300.0 s This will raise an error if the method is unsupported: >>> template = Template().construct( ... method='from_meta_file', name='test', lowcut=2.0, highcut=8.0, ... samp_rate=20.0, filt_order=4, prepick=0.1, swin='all', ... length=2.0) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): NotImplementedError: Method is not supported, use \ Tribe.construct instead. """ if method in [ 'from_meta_file', 'from_seishub', 'from_client', 'multi_template_gen' ]: raise NotImplementedError('Method is not supported, ' 'use Tribe.construct instead.') streams, events, process_lengths = template_gen.template_gen( method=method, lowcut=lowcut, highcut=highcut, length=length, filt_order=filt_order, samp_rate=samp_rate, prepick=prepick, return_event=True, swin=swin, process_len=process_len, all_horiz=all_horiz, delayed=delayed, plot=plot, plotdir=plotdir, min_snr=min_snr, parallel=parallel, num_cores=num_cores, skip_short_chans=skip_short_chans, **kwargs) self.name = name st = streams[0] event = events[0] process_length = process_lengths[0] for tr in st: if not np.any(tr.data.astype(np.float16)): Logger.warning('Data are zero in float16, missing data,' ' will not use: {0}'.format(tr.id)) st.remove(tr) self.st = st self.lowcut = lowcut self.highcut = highcut self.filt_order = filt_order self.samp_rate = samp_rate self.process_length = process_length self.prepick = prepick self.event = event return self
def test_debug_levels(self): template = template_gen(self.picks, self.st.copy(), 10, debug=3) self.assertEqual(len(template), len(self.picks))
def construct(self, method, lowcut, highcut, samp_rate, filt_order, length, prepick, swin="all", process_len=86400, all_horiz=False, delayed=True, plot=False, plotdir=None, min_snr=None, parallel=False, num_cores=False, skip_short_chans=False, save_progress=False, **kwargs): """ Generate a Tribe of Templates. :type method: str :param method: Method of Tribe generation. Possible options are: `from_client`, `from_seishub`, `from_meta_file`. See below on the additional required arguments for each method. :type lowcut: float :param lowcut: Low cut (Hz), if set to None will not apply a lowcut :type highcut: float :param highcut: High cut (Hz), if set to None will not apply a highcut. :type samp_rate: float :param samp_rate: New sampling rate in Hz. :type filt_order: int :param filt_order: Filter level (number of corners). :type length: float :param length: Length of template waveform in seconds. :type prepick: float :param prepick: Pre-pick time in seconds :type swin: str :param swin: P, S, P_all, S_all or all, defaults to all: see note in :func:`eqcorrscan.core.template_gen.template_gen` :type process_len: int :param process_len: Length of data in seconds to download and process. :type all_horiz: bool :param all_horiz: To use both horizontal channels even if there is only a pick on one of them. Defaults to False. :type delayed: bool :param delayed: If True, each channel will begin relative to it's own pick-time, if set to False, each channel will begin at the same time. :type plot: bool :param plot: Plot templates or not. :type plotdir: str :param plotdir: The path to save plots to. If `plotdir=None` (default) then the figure will be shown on screen. :type min_snr: float :param min_snr: Minimum signal-to-noise ratio for a channel to be included in the template, where signal-to-noise ratio is calculated as the ratio of the maximum amplitude in the template window to the rms amplitude in the whole window given. :type parallel: bool :param parallel: Whether to process data in parallel or not. :type num_cores: int :param num_cores: Number of cores to try and use, if False and parallel=True, will use either all your cores, or as many traces as in the data (whichever is smaller). :type save_progress: bool :param save_progress: Whether to save the resulting template set at every data step or not. Useful for long-running processes. :type skip_short_chans: bool :param skip_short_chans: Whether to ignore channels that have insufficient length data or not. Useful when the quality of data is not known, e.g. when downloading old, possibly triggered data from a datacentre :type save_progress: bool :param save_progress: Whether to save the resulting party at every data step or not. Useful for long-running processes. .. note:: *Method specific arguments:* - `from_client` requires: :param str client_id: string passable by obspy to generate Client, or any object with a `get_waveforms` method, including a Client instance. :param `obspy.core.event.Catalog` catalog: Catalog of events to generate template for :param float data_pad: Pad length for data-downloads in seconds - `from_seishub` requires: :param str url: url to seishub database :param `obspy.core.event.Catalog` catalog: Catalog of events to generate template for :param float data_pad: Pad length for data-downloads in seconds - `from_meta_file` requires: :param str meta_file: Path to obspy-readable event file, or an obspy Catalog :param `obspy.core.stream.Stream` st: Stream containing waveform data for template. Note that this should be the same length of stream as you will use for the continuous detection, e.g. if you detect in day-long files, give this a day-long file! :param bool process: Whether to process the data or not, defaults to True. .. Note:: Method: `from_sac` is not supported by Tribe.construct and must use Template.construct. .. Note:: Templates will be named according to their start-time. """ templates, catalog, process_lengths = template_gen.template_gen( method=method, lowcut=lowcut, highcut=highcut, length=length, filt_order=filt_order, samp_rate=samp_rate, prepick=prepick, return_event=True, save_progress=save_progress, swin=swin, process_len=process_len, all_horiz=all_horiz, plotdir=plotdir, delayed=delayed, plot=plot, min_snr=min_snr, parallel=parallel, num_cores=num_cores, skip_short_chans=skip_short_chans, **kwargs) for template, event, process_len in zip(templates, catalog, process_lengths): t = Template() for tr in template: if not np.any(tr.data.astype(np.float16)): Logger.warning('Data are zero in float16, missing data,' ' will not use: {0}'.format(tr.id)) template.remove(tr) if len(template) == 0: Logger.error('Empty Template') continue t.st = template t.name = template.sort(['starttime'])[0]. \ stats.starttime.strftime('%Y_%m_%dt%H_%M_%S') t.lowcut = lowcut t.highcut = highcut t.filt_order = filt_order t.samp_rate = samp_rate t.process_length = process_len t.prepick = prepick event.comments.append(Comment( text="eqcorrscan_template_" + t.name, creation_info=CreationInfo(agency='eqcorrscan', author=getpass.getuser()))) t.event = event self.templates.append(t) return self
def run_tutorial(min_magnitude=2, shift_len=0.2, num_cores=4, min_cc=0.5): """Functional, tested example script for running the lag-calc tutorial.""" if num_cores > cpu_count(): num_cores = cpu_count() client = Client('NCEDC') t1 = UTCDateTime(2004, 9, 28) t2 = t1 + 86400 print('Downloading catalog') catalog = client.get_events(starttime=t1, endtime=t2, minmagnitude=min_magnitude, minlatitude=35.7, maxlatitude=36.1, minlongitude=-120.6, maxlongitude=-120.2, includearrivals=True) # We don't need all the picks, lets take the information from the # five most used stations - note that this is done to reduce computational # costs. catalog = catalog_utils.filter_picks(catalog, channels=['EHZ'], top_n_picks=5) # There is a duplicate pick in event 3 in the catalog - this has the effect # of reducing our detections - check it yourself. for pick in catalog[3].picks: if pick.waveform_id.station_code == 'PHOB' and \ pick.onset == 'emergent': catalog[3].picks.remove(pick) print('Generating templates') templates = template_gen.template_gen(method="from_client", catalog=catalog, client_id='NCEDC', lowcut=2.0, highcut=9.0, samp_rate=50.0, filt_order=4, length=3.0, prepick=0.15, swin='all', process_len=3600) # In this section we generate a series of chunks of data. start_time = UTCDateTime(2004, 9, 28, 17) end_time = UTCDateTime(2004, 9, 28, 20) process_len = 3600 chunks = [] chunk_start = start_time while chunk_start < end_time: chunk_end = chunk_start + process_len if chunk_end > end_time: chunk_end = end_time chunks.append((chunk_start, chunk_end)) chunk_start += process_len all_detections = [] picked_catalog = Catalog() template_names = [ template[0].stats.starttime.strftime("%Y%m%d_%H%M%S") for template in templates ] for t1, t2 in chunks: print('Downloading and processing for start-time: %s' % t1) # Download and process the data bulk_info = [(tr.stats.network, tr.stats.station, '*', tr.stats.channel, t1, t2) for tr in templates[0]] # Just downloading a chunk of data try: st = client.get_waveforms_bulk(bulk_info) except FDSNException: st = Stream() for _bulk in bulk_info: st += client.get_waveforms(*_bulk) st.merge(fill_value='interpolate') st = pre_processing.shortproc(st, lowcut=2.0, highcut=9.0, filt_order=4, samp_rate=50.0, num_cores=num_cores) detections = match_filter.match_filter(template_names=template_names, template_list=templates, st=st, threshold=8.0, threshold_type='MAD', trig_int=6.0, plotvar=False, plotdir='.', cores=num_cores) # Extract unique detections from set. unique_detections = [] for master in detections: keep = True for slave in detections: if not master == slave and\ abs(master.detect_time - slave.detect_time) <= 1.0: # If the events are within 1s of each other then test which # was the 'best' match, strongest detection if not master.detect_val > slave.detect_val: keep = False break if keep: unique_detections.append(master) all_detections += unique_detections picked_catalog += lag_calc.lag_calc(detections=unique_detections, detect_data=st, template_names=template_names, templates=templates, shift_len=shift_len, min_cc=min_cc, interpolate=False, plot=False) # Return all of this so that we can use this function for testing. return all_detections, picked_catalog, templates, template_names