def match_filter(template_names,
template_list,
st,
threshold,
threshold_type,
trig_int,
plotvar,
plotdir='.',
cores=1,
tempdir=False,
debug=0,
plot_format='jpg'):
r"""Over-arching code to run the correlations of given templates with a\
day of seismic data and output the detections based on a given threshold.
:type template_names: list
:param template_names: List of template names in the same order as\
template_list
:type template_list: list :class: 'obspy.Stream'
:param template_list: A list of templates of which each template is a\
Stream of obspy traces containing seismic data and header information.
:type st: :class: 'obspy.Stream'
:param st: An obspy.Stream object containing all the data available and\
required for the correlations with templates given. For efficiency\
this should contain no excess traces which are not in one or more of\
the templates. This will now remove excess traces internally, but\
will copy the stream and work on the copy, leaving your input stream\
untouched.
:type threshold: float
:param threshold: A threshold value set based on the threshold_type
:type threshold_type: str
:param threshold_type: The type of threshold to be used, can be MAD,\
absolute or av_chan_corr. MAD threshold is calculated as the\
threshold*(median(abs(cccsum))) where cccsum is the cross-correlation\
sum for a given template. absolute threhsold is a true absolute\
threshold based on the cccsum value av_chan_corr is based on the mean\
values of single-channel cross-correlations assuming all data are\
present as required for the template, \
e.g. av_chan_corr_thresh=threshold*(cccsum/len(template)) where\
template is a single template from the input and the length is the\
number of channels within this template.
:type trig_int: float
:param trig_int: Minimum gap between detections in seconds.
:type plotvar: bool
:param plotvar: Turn plotting on or off
:type plotdir: str
:param plotdir: Path to plotting folder, plots will be output here,\
defaults to run location.
:type tempdir: String or False
:param tempdir: Directory to put temporary files, or False
:type cores: int
:param cores: Number of cores to use
:type debug: int
:param debug: Debug output level, the bigger the number, the more the\
output.
:return: :class: 'DETECTIONS' detections for each channel formatted as\
:class: 'obspy.UTCDateTime' objects.
.. rubric:: Note
Plotting within the match-filter routine uses the Agg backend with\
interactive plotting turned off. This is because the function is\
designed to work in bulk. If you wish to turn interactive plotting on\
you must import matplotlib in your script first, when you them import\
match_filter you will get the warning that this call to matplotlib has\
no effect, which will mean that match_filter has not changed the\
plotting behaviour.
"""
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.ioff()
import copy
from eqcorrscan.utils import EQcorrscan_plotting
from eqcorrscan.utils import findpeaks
from obspy import Trace
import time
# Copy the stream here because we will f**k about with it
stream = st.copy()
templates = copy.deepcopy(template_list)
# Debug option to confirm that the channel names match those in the
# templates
if debug >= 2:
template_stachan = []
data_stachan = []
for template in templates:
for tr in template:
template_stachan.append(tr.stats.station + '.' +
tr.stats.channel)
for tr in stream:
data_stachan.append(tr.stats.station + '.' + tr.stats.channel)
template_stachan = list(set(template_stachan))
data_stachan = list(set(data_stachan))
if debug >= 3:
print 'I have template info for these stations:'
print template_stachan
print 'I have daylong data for these stations:'
print data_stachan
# Perform a check that the daylong vectors are daylong
for tr in stream:
if not tr.stats.sampling_rate * 86400 == tr.stats.npts:
msg = ' '.join([
'Data are not daylong for', tr.stats.station, tr.stats.channel
])
raise ValueError(msg)
# Call the _template_loop function to do all the correlation work
outtic = time.clock()
# Edit here from previous, stable, but slow match_filter
# Would be worth testing without an if statement, but with every station in
# the possible template stations having data, but for those without real
# data make the data NaN to return NaN ccc_sum
# Note: this works
if debug >= 2:
print 'Ensuring all template channels have matches in daylong data'
template_stachan = []
for template in templates:
for tr in template:
template_stachan += [(tr.stats.station, tr.stats.channel)]
template_stachan = list(set(template_stachan))
# Copy this here to keep it safe
for stachan in template_stachan:
if not stream.select(station=stachan[0], channel=stachan[1]):
# Remove template traces rather than adding NaN data
for template in templates:
if template.select(station=stachan[0], channel=stachan[1]):
for tr in template.select(station=stachan[0],
channel=stachan[1]):
template.remove(tr)
# Remove un-needed channels
for tr in stream:
if not (tr.stats.station, tr.stats.channel) in template_stachan:
stream.remove(tr)
# Also pad out templates to have all channels
for template in templates:
for stachan in template_stachan:
if not template.select(station=stachan[0], channel=stachan[1]):
nulltrace = Trace()
nulltrace.stats.station = stachan[0]
nulltrace.stats.channel = stachan[1]
nulltrace.stats.sampling_rate = template[0].stats.sampling_rate
nulltrace.stats.starttime = template[0].stats.starttime
nulltrace.data = np.array([np.NaN] * len(template[0].data),
dtype=np.float32)
template += nulltrace
if debug >= 2:
print 'Starting the correlation run for this day'
[cccsums, no_chans] = _channel_loop(templates, stream, cores, debug)
if len(cccsums[0]) == 0:
raise ValueError('Correlation has not run, zero length cccsum')
outtoc = time.clock()
print ' '.join([
'Looping over templates and streams took:',
str(outtoc - outtic), 's'
])
if debug >= 2:
print ' '.join(
['The shape of the returned cccsums is:',
str(np.shape(cccsums))])
print ' '.join(['This is from', str(len(templates)), 'templates'])
print ' '.join(
['Correlated with',
str(len(stream)), 'channels of data'])
detections = []
for i, cccsum in enumerate(cccsums):
template = templates[i]
if threshold_type == 'MAD':
rawthresh = threshold * np.median(np.abs(cccsum))
elif threshold_type == 'absolute':
rawthresh = threshold
elif threshold == 'av_chan_corr':
rawthresh = threshold * (cccsum / len(template))
else:
print 'You have not selected the correct threshold type, I will' +\
'use MAD as I like it'
rawthresh = threshold * np.mean(np.abs(cccsum))
# Findpeaks returns a list of tuples in the form [(cccsum, sample)]
print ' '.join(['Threshold is set at:', str(rawthresh)])
print ' '.join(['Max of data is:', str(max(cccsum))])
print ' '.join(['Mean of data is:', str(np.mean(cccsum))])
if np.abs(np.mean(cccsum)) > 0.05:
warnings.warn('Mean is not zero! Check this!')
# Set up a trace object for the cccsum as this is easier to plot and
# maintins timing
if plotvar:
stream_plot = copy.deepcopy(stream[0])
# Downsample for plotting
stream_plot.decimate(int(stream[0].stats.sampling_rate / 10))
cccsum_plot = Trace(cccsum)
cccsum_plot.stats.sampling_rate = stream[0].stats.sampling_rate
# Resample here to maintain shape better
cccsum_hist = cccsum_plot.copy()
cccsum_hist = cccsum_hist.decimate(
int(stream[0].stats.sampling_rate / 10)).data
cccsum_plot = EQcorrscan_plotting.chunk_data(
cccsum_plot, 10, 'Maxabs').data
# Enforce same length
stream_plot.data = stream_plot.data[0:len(cccsum_plot)]
cccsum_plot = cccsum_plot[0:len(stream_plot.data)]
cccsum_hist = cccsum_hist[0:len(stream_plot.data)]
EQcorrscan_plotting.triple_plot(
cccsum_plot, cccsum_hist, stream_plot, rawthresh, True,
plotdir + '/cccsum_plot_' + template_names[i] + '_' +
stream[0].stats.starttime.datetime.strftime('%Y-%m-%d') + '.' +
plot_format)
if debug >= 4:
print ' '.join([
'Saved the cccsum to:', template_names[i],
stream[0].stats.starttime.datetime.strftime('%Y%j')
])
np.save(
template_names[i] +
stream[0].stats.starttime.datetime.strftime('%Y%j'),
cccsum)
tic = time.clock()
if debug >= 4:
np.save('cccsum_' + str(i) + '.npy', cccsum)
if debug >= 3 and max(cccsum) > rawthresh:
peaks = findpeaks.find_peaks2_short(
cccsum, rawthresh, trig_int * stream[0].stats.sampling_rate,
debug, stream[0].stats.starttime,
stream[0].stats.sampling_rate)
elif max(cccsum) > rawthresh:
peaks = findpeaks.find_peaks2_short(
cccsum, rawthresh, trig_int * stream[0].stats.sampling_rate,
debug)
else:
print 'No peaks found above threshold'
peaks = False
toc = time.clock()
if debug >= 1:
print ' '.join(['Finding peaks took:', str(toc - tic), 's'])
if peaks:
for peak in peaks:
detecttime = stream[0].stats.starttime +\
peak[1] / stream[0].stats.sampling_rate
detections.append(
DETECTION(template_names[i], detecttime, no_chans[i],
peak[0], rawthresh, 'corr'))
del stream, templates
return detections
stream=Stream(stream)
print stream
if not Prep:
#stream_copy=stream.copy() # Keep the stream safe
print "Running the detection routine"
# Check that the data are okay
detect_templates, detect_nodes=bright_lights.brightness(stations, \
nodes, lags, stream,
brightdef.threshold, brightdef.thresh_type,\
brightdef.coherance, instance, matchdef, templatedef,\
brightdef)
del detect_templates#, stream # Delete templates from memory to conserve RAM!
#stream=stream_copy
nodesout+=detect_nodes
if Split:
plotting.threeD_gridplot(nodesout, save=brightdef.plotsave,\
savefile='Detected_nodes_'+str(instance)+'.png')
else:
plotting.threeD_gridplot(nodesout, save=brightdef.plotsave,\
savefile='Detected_nodes.png')
else:
for tr in stream:
print "Writing data as: test_data/"+tr.stats.station+'-'+tr.stats.channel+\
'-'+str(tr.stats.starttime.year)+\
'-'+str(tr.stats.starttime.month).zfill(2)+\
'-'+str(tr.stats.starttime.day).zfill(2)+\
'-processed.ms'
tr.write('test_data/'+tr.stats.station+'-'+tr.stats.channel+\
'-'+str(tr.stats.starttime.year)+\
'-'+str(tr.stats.starttime.month).zfill(2)+\
'-'+str(tr.stats.starttime.day).zfill(2)+\
def match_filter(
template_names, templates, stream, threshold, threshold_type, trig_int, plotvar, cores=1, tempdir=False, debug=0
):
"""
Over-arching code to run the correlations of given templates with a day of
seismic data and output the detections based on a given threshold.
:type templates: list :class: 'obspy.Stream'
:param templates: A list of templates of which each template is a Stream of\
obspy traces containing seismic data and header information.
:type stream: :class: 'obspy.Stream'
:param stream: An obspy.Stream object containing all the data available and\
required for the correlations with templates given. For efficiency this\
should contain no excess traces which are not in one or more of the\
templates.
:type threshold: float
:param threshold: A threshold value set based on the threshold_type
:type threshold_type: str
:param threshold_type: The type of threshold to be used, can be MAD,\
absolute or av_chan_corr. MAD threshold is calculated as the\
threshold*(median(abs(cccsum))) where cccsum is the cross-correlation\
sum for a given template. absolute threhsold is a true absolute\
threshold based on the cccsum value av_chan_corr is based on the mean\
values of single-channel cross-correlations assuming all data are\
present as required for the template, \
e.g. av_chan_corr_thresh=threshold*(cccsum/len(template)) where\
template is a single template from the input and the length is the\
number of channels within this template.
:type trig_int: float
:param trig_int: Minimum gap between detections in seconds.
:type tempdir: String or False
:param tempdir: Directory to put temporary files, or False
:type cores: int
:param cores: Number of cores to use
:type debug: int
:param debug: Debug output level, the bigger the number, the more the output
:return: :class: 'DETECTIONS' detections for each channel formatted as\
:class: 'obspy.UTCDateTime' objects.
"""
from eqcorrscan.utils import findpeaks, EQcorrscan_plotting
import time, copy
from obspy import Trace
match_internal = False # Set to True if memory is an issue, if True, will only
# use about the same amount of memory as the seismic dat
# take up. If False, it will use 20-100GB per instance
# Debug option to confirm that the channel names match those in the templates
if debug >= 2:
template_stachan = []
data_stachan = []
for template in templates:
for tr in template:
template_stachan.append(tr.stats.station + "." + tr.stats.channel)
for tr in stream:
data_stachan.append(tr.stats.station + "." + tr.stats.channel)
template_stachan = list(set(template_stachan))
data_stachan = list(set(data_stachan))
if debug >= 3:
print "I have template info for these stations:"
print template_stachan
print "I have daylong data for these stations:"
print data_stachan
# Perform a check that the daylong vectors are daylong
for tr in stream:
if not tr.stats.sampling_rate * 86400 == tr.stats.npts:
raise ValueError("Data are not daylong for " + tr.stats.station + "." + tr.stats.channel)
# Call the _template_loop function to do all the correlation work
outtic = time.clock()
# Edit here from previous, stable, but slow match_filter
# Would be worth testing without an if statement, but with every station in
# the possible template stations having data, but for those without real
# data make the data NaN to return NaN ccc_sum
if debug >= 2:
print "Ensuring all template channels have matches in daylong data"
template_stachan = []
for template in templates:
for tr in template:
template_stachan += [(tr.stats.station, tr.stats.channel)]
template_stachan = list(set(template_stachan))
# Copy this here to keep it safe
for stachan in template_stachan:
if not stream.select(station=stachan[0], channel=stachan[1]):
# Add a trace of NaN's
nulltrace = Trace()
nulltrace.stats.station = stachan[0]
nulltrace.stats.channel = stachan[1]
nulltrace.stats.sampling_rate = stream[0].stats.sampling_rate
nulltrace.stats.starttime = stream[0].stats.starttime
nulltrace.data = np.array([np.NaN] * len(stream[0].data), dtype=np.float32)
stream += nulltrace
# Also pad out templates to have all channels
for template in templates:
for stachan in template_stachan:
if not template.select(station=stachan[0], channel=stachan[1]):
nulltrace = Trace()
nulltrace.stats.station = stachan[0]
nulltrace.stats.channel = stachan[1]
nulltrace.stats.sampling_rate = template[0].stats.sampling_rate
nulltrace.stats.starttime = template[0].stats.starttime
nulltrace.data = np.array([np.NaN] * len(template[0].data), dtype=np.float32)
template += nulltrace
if debug >= 2:
print "Starting the correlation run for this day"
if match_internal:
[cccsums, no_chans] = run_channel_loop(templates, stream, tempdir)
else:
[cccsums, no_chans] = _channel_loop(templates, stream, cores, debug)
if len(cccsums[0]) == 0:
raise ValueError("Correlation has not run, zero length cccsum")
outtoc = time.clock()
print "Looping over templates and streams took: " + str(outtoc - outtic) + " s"
if debug >= 2:
print "The shape of the returned cccsums is: " + str(np.shape(cccsums))
print "This is from " + str(len(templates)) + " templates"
print "Correlated with " + str(len(stream)) + " channels of data"
i = 0
detections = []
for cccsum in cccsums:
template = templates[i]
if threshold_type == "MAD":
rawthresh = threshold * np.median(np.abs(cccsum))
elif threshold_type == "absolute":
rawthresh = threshold
elif threshold == "av_chan_corr":
rawthresh = threshold * (cccsum / len(template))
else:
print "You have not selected the correct threshold type, I will use MAD as I like it"
rawthresh = threshold * np.mean(np.abs(cccsum))
# Findpeaks returns a list of tuples in the form [(cccsum, sample)]
print "Threshold is set at: " + str(rawthresh)
print "Max of data is: " + str(max(cccsum))
print "Mean of data is: " + str(np.mean(cccsum))
if np.abs(np.mean(cccsum)) > 0.05:
warnings.warn("Mean is not zero! Check this!")
# Set up a trace object for the cccsum as this is easier to plot and
# maintins timing
if plotvar:
stream_plot = copy.deepcopy(stream[0])
# Downsample for plotting
stream_plot.decimate(int(stream[0].stats.sampling_rate / 20))
cccsum_plot = Trace(cccsum)
cccsum_plot.stats.sampling_rate = stream[0].stats.sampling_rate
# Resample here to maintain shape better
cccsum_hist = cccsum_plot.copy()
cccsum_hist = cccsum_hist.decimate(int(stream[0].stats.sampling_rate / 20)).data
cccsum_plot = EQcorrscan_plotting.chunk_data(cccsum_plot, 20, "Maxabs").data
# Enforce same length
stream_plot.data = stream_plot.data[0 : len(cccsum_plot)]
cccsum_plot = cccsum_plot[0 : len(stream_plot.data)]
cccsum_hist = cccsum_hist[0 : len(stream_plot.data)]
EQcorrscan_plotting.triple_plot(
cccsum_plot,
cccsum_hist,
stream_plot,
rawthresh,
True,
"plot/cccsum_plot_"
+ template_names[i]
+ "_"
+ str(stream[0].stats.starttime.year)
+ "-"
+ str(stream[0].stats.starttime.month)
+ "-"
+ str(stream[0].stats.starttime.day)
+ ".jpg",
)
np.save(template_names[i] + stream[0].stats.starttime.datetime.strftime("%Y%j"), cccsum)
tic = time.clock()
if debug >= 4:
np.save("cccsum_" + str(i) + ".npy", cccsum)
if debug >= 3 and max(cccsum) > rawthresh:
peaks = findpeaks.find_peaks2_short(
cccsum,
rawthresh,
trig_int * stream[0].stats.sampling_rate,
debug,
stream[0].stats.starttime,
stream[0].stats.sampling_rate,
)
elif max(cccsum) > rawthresh:
peaks = findpeaks.find_peaks2_short(cccsum, rawthresh, trig_int * stream[0].stats.sampling_rate, debug)
else:
print "No peaks found above threshold"
peaks = False
toc = time.clock()
if debug >= 1:
print "Finding peaks took: " + str(toc - tic) + " s"
if peaks:
for peak in peaks:
detecttime = stream[0].stats.starttime + peak[1] / stream[0].stats.sampling_rate
detections.append(DETECTION(template_names[i], detecttime, no_chans[i], peak[0], rawthresh, "corr"))
i += 1
return detections
def test_match_filter(samp_rate=20.0, debug=0):
"""
Function to test the capabilities of match_filter and just check that it\
is working! Uses synthetic templates and seeded, randomised data.
:type debug: int
:param debug: Debug level, higher the number the more output.
"""
from eqcorrscan.utils import pre_processing
from eqcorrscan.utils import EQcorrscan_plotting as plotting
from eqcorrscan.core import match_filter
from obspy import UTCDateTime
import string
# Generate a random dataset
templates, data, seeds = generate_synth_data(nsta=5, ntemplates=2,
nseeds=50,
samp_rate=samp_rate,
t_length=6.0, max_amp=5.0,
debug=debug)
# Notes to the user: If you use more templates you should ensure they are
# more different, e.g. set the data to have larger moveouts, otherwise
# similar templates will detect events seeded by another template.
# Test the pre_processing functions
for tr in data:
pre_processing.dayproc(tr=tr, lowcut=2.0, highcut=8.0, filt_order=3,
samp_rate=20.0, debug=0,
starttime=UTCDateTime(0))
if debug > 0:
data.plot()
# Filter the data and the templates
for template in templates:
pre_processing.shortproc(st=template, lowcut=2.0, highcut=8.0,
filt_order=3, samp_rate=20.0)
if debug > 0:
template.plot()
template_names = list(string.ascii_lowercase)[0:len(templates)]
detections = match_filter.match_filter(template_names=template_names,
template_list=templates,
st=data, threshold=8.0,
threshold_type='MAD',
trig_int=6.0,
plotvar=False,
plotdir='.',
cores=1)
# Compare the detections to the seeds
print('This test made ' + str(len(detections)) + ' detections')
ktrue = 0
kfalse = 0
for detection in detections:
print(detection.template_name)
i = template_names.index(detection.template_name)
t_seeds = seeds[i]
dtime_samples = int((detection.detect_time - UTCDateTime(0)) *
samp_rate)
if dtime_samples in t_seeds['time']:
j = list(t_seeds['time']).index(dtime_samples)
print('Detection at SNR of: ' + str(t_seeds['SNR'][j]))
ktrue += 1
else:
min_diff = min(abs(t_seeds['time'] - dtime_samples))
if min_diff < 10:
# If there is a match within ten samples then its good enough
j = list(abs(t_seeds['time'] - dtime_samples)).index(min_diff)
print('Detection at SNR of: ' + str(t_seeds['SNR'][j]))
ktrue += 1
else:
print('Detection at sample: ' + str(dtime_samples) +
' does not match anything in seed times:')
kfalse += 1
print('Minimum difference in samples is: ' + str(min_diff))
# Plot the detections
if debug > 3:
for i, template in enumerate(templates):
times = [d.detect_time.datetime for d in detections
if d.template_name == template_names[i]]
print(times)
plotting.detection_multiplot(data, template, times)
# Set an 'acceptable' ratio of positive to false detections
print(str(ktrue) + ' true detections and ' + str(kfalse) +
' false detections')
if kfalse / ktrue < 0.25:
return True
else:
return False
def find_peaks_dep(arr,
thresh,
trig_int,
debug=0,
starttime=False,
samp_rate=1.0):
r"""Function to determine peaks in an array of data above a certain \
threshold.
Depreciated peak-finding routine, very slow, but accurate. If all else \
fails this one should work.
:type arr: ndarray
:param arr: 1-D numpy array is required
:type thresh: float
:param thresh: The threshold below which will be considered noise and \
peaks will not be found in.
:type trig_int: int
:param trig_int: The minimum difference in samples between triggers,\
if multiple peaks within this window this code will find the highest.
:type starttime: osbpy.UTCDateTime
:param starttime: Starttime for plotting, only used if debug > 2.
:type samp_rate: float
:param samp_rate: Sampling rate in Hz, only used for plotting if debug > 2.
:return: peaks: Lists of tuples of peak values and locations.
"""
import numpy as np
from obspy import UTCDateTime
if not starttime:
starttime = UTCDateTime(0)
# Perform some checks
if trig_int < 3:
msg = 'Trigger interval must be greater than 2 samples to find maxima'
raise IOError(msg)
# from joblib import Parallel, delayed
# Will find peaks in the absolute then transfer these to the true values
sig = np.abs(arr) - thresh
true_peaks = []
for i in xrange(int(trig_int), int(len(sig) - trig_int), int(trig_int)):
window = sig[i - trig_int:i + trig_int]
# Define a moving window containing data from +/- the trigger iterval
peaks = []
locs = []
for j in xrange(1, len(window) - 1):
# Find all turning points within the window
if window[j] > 0.0 and window[j] > window[j+1] and\
window[j] > window[j - 1]:
peaks.append(window[j])
locs.append(i - trig_int + j)
# Find maximum peak in window
if peaks:
true_peaks.append(
(np.max(np.array(peaks)), locs[np.argmax(np.array(peaks))]))
# Get unique values
peaks = sorted(list(set(true_peaks)), key=lambda loc: loc[1])
# Find highest peak in peaks within trig_int of each other
for i in xrange(1, len(peaks) - 1):
if peaks[i + 1][1] - peaks[i][1] < trig_int:
if peaks[i][0] < peaks[i + 1][0]:
peaks[i] = peaks[i + 1]
else:
peaks[i + 1] = peaks[i]
elif peaks[i][1] - peaks[i - 1][1] < trig_int:
if peaks[i][0] < peaks[i - 1][0]:
peaks[i] = peaks[i - 1]
else:
peaks[i - 1] = peaks[i]
peaks = sorted(list(set(peaks)), key=lambda loc: loc[1])
if debug >= 3:
from eqcorrscan.utils import EQcorrscan_plotting
_fname = ''.join(
['peaks_',
starttime.datetime.strftime('%Y-%m-%d'), '.pdf'])
EQcorrscan_plotting.peaks_plot(arr, starttime, samp_rate, True, peaks,
_fname)
return peaks
def find_peaks2(arr,
thresh,
trig_int,
debug=0,
maxwidth=10,
starttime=False,
samp_rate=1.0):
r"""Function to determine peaks in an array of data using scipy \
find_peaks_cwt, works fast in certain cases, but for match_filter cccsum \
peak finding, find_peaks2_short works better. Test it out and see which \
works best for your application.
:type arr: ndarray
:param arr: 1-D numpy array is required
:type thresh: float
:param thresh: The threshold below which will be considered noise and \
peaks will not be found in.
:type trig_int: int
:param trig_int: The minimum difference in samples between triggers, \
if multiple peaks within this window this code will find the highest.
:type debug: int
:param debug: Optional, debug level 0-5
:type maxwidth: int
:param maxwidth: Maximum peak width to look for in samples
:type starttime: osbpy.UTCDateTime
:param starttime: Starttime for plotting, only used if debug > 2.
:type samp_rate: float
:param samp_rate: Sampling rate in Hz, only used for plotting if debug > 2.
:return: peaks: Lists of tuples of peak values and locations.
"""
from scipy.signal import find_peaks_cwt
import numpy as np
from obspy import UTCDateTime
if not starttime:
starttime = UTCDateTime(0)
# Set everything below the threshold to zero
image = np.copy(arr)
image = np.abs(image)
image[image < thresh] = thresh
# We need to check if the number of samples in the image is prime, if it
# is this method will be really slow, so we add a pad to the end to make
# it not of prime length!
if is_prime(len(image)):
image = np.append(image, 0.0)
print('Input array has a prime number of samples, appending a zero')
print(len(image))
if len(image[image > thresh]) == 0:
print('No values over threshold found')
return []
if debug > 0:
msg = ' '.join([
'Found',
str(len(image[image > thresh])), 'samples above the threshold'
])
print(msg)
initial_peaks = []
peaks = []
# Find the peaks
print('Finding peaks')
peakinds = find_peaks_cwt(image, np.arange(1, maxwidth))
initial_peaks = [(image[peakind], peakind) for peakind in peakinds]
# Sort initial peaks according to amplitude
print('sorting peaks')
peaks_sort = sorted(initial_peaks,
key=lambda amplitude: amplitude[0],
reverse=True)
if debug >= 4:
for peak in initial_peaks:
print(peak)
if initial_peaks:
peaks.append(peaks_sort[0]) # Definitely take the biggest peak
if debug > 3:
msg = ' '.join([
'Added the biggest peak of',
str(peaks[0][0]), 'at sample',
str(peaks[0][1])
])
print(msg)
if len(initial_peaks) > 1:
if debug > 3:
msg = ' '.join([
'Multiple peaks found, checking them',
'now to see if they overlap'
])
print(msg)
for next_peak in peaks_sort:
# i in xrange(1,len(peaks_sort)):
# Loop through the amplitude sorted peaks
# if the next highest amplitude peak is within trig_int of any
# peak already in peaks then we don't want it, else, add it
# next_peak = peaks_sort[i]
if debug > 3:
print(next_peak)
for peak in peaks:
add = False
# Use add as a switch for whether or not to append
# next peak to peaks, if once gone through all the peaks
# it is True, then we will add it, otherwise we won't!
if abs(next_peak[1] - peak[1]) < trig_int:
if debug > 3:
msg = ' '.join([
'Difference in time is',
str(next_peak[1] - peak[1]), '\n'
'Which is less than',
str(trig_int)
])
print(msg)
add = False
# Need to exit the loop here if false
break
else:
add = True
if add:
if debug > 3:
msg = ' '.join([
'Adding peak of',
str(next_peak[0]), 'at sample',
str(next_peak[1])
])
print(msg)
peaks.append(next_peak)
elif debug > 3:
msg = ' '.join([
'I did not add peak of',
str(next_peak[0]), 'at sample',
str(next_peak[1])
])
print(msg)
if debug >= 3:
from eqcorrscan.utils import EQcorrscan_plotting
_fname = ''.join(
['peaks_',
starttime.datetime.strftime('%Y-%m-%d'), '.pdf'])
print(' '.join(['Saving plot to', _fname]))
EQcorrscan_plotting.peaks_plot(image, starttime, samp_rate, True,
peaks, _fname)
peaks = sorted(peaks, key=lambda time: time[1], reverse=False)
return peaks
else:
print('No peaks for you!')
return peaks
# Now lets try and work out how many unique events we have just to compare
# with the GeoNet catalog of 20 events on this day in this sequence
for master in detections:
keep = True
for slave in detections:
if not master == slave and\
abs(master.detect_time - slave.detect_time) <= 6.0:
# If the events are within 6s of each other then test which
# was the 'best' match, strongest detection
if not master.detect_val > slave.detect_val:
keep = False
break
if keep:
unique_detections.append(master)
print('We made a total of ' + str(len(unique_detections)) + ' detections')
for detection in unique_detections:
print('Detection at :' + str(detection.detect_time) + ' for template ' +
detection.template_name + ' with a cross-correlation sum of: ' +
str(detection.detect_val))
# We can plot these too
stplot = st.copy()
template = templates[template_names.index(detection.template_name)]
lags = sorted([tr.stats.starttime for tr in template])
maxlag = lags[-1] - lags[0]
stplot.trim(starttime=detection.detect_time - 10,
endtime=detection.detect_time + maxlag + 10)
plotting.detection_multiplot(stplot, template,
[detection.detect_time.datetime])
stream = obsread('scripts/brightness_test_daylong.ms')
stream.trim(starttime=UTCDateTime('2011-09-04 17:05:00'),\
endtime=UTCDateTime('2011-09-04 17:15:00'))#, pad=True,\
# fill_value=0)
# for tr in stream:
# if tr.stats.station=='WVZ':
# stream.remove(tr)
stream.filter('bandpass', freqmin=4.0, freqmax=8.0)
# stream.trim(stream[0].stats.starttime+90, stream[0].stats.endtime)
stream.trim(stream[0].stats.starttime,
stream[0].stats.endtime,
pad=True,
fill_value=0)
stream.plot(size=(800, 600), equal_scale=False)
instance = 0
# Cut the nodes...
cutnodes = [nodes[0]] + [nodes[116]]
cutlags = np.array([lags[:, 0]] + [lags[:, 116]]).T
detect_templates, detect_nodes=bright_lights.brightness(stations, \
nodes, lags, stream,
brightdef.threshold, brightdef.thresh_type,\
brightdef.coherance, instance, matchdef, templatedef,\
brightdef)
plotting.threeD_gridplot(detect_nodes)
# detect_templates, detect_nodes=bright_lights.brightness(stations, \
# cutnodes, cutlags, stream,
# brightdef.threshold, brightdef.thresh_type,\
# brightdef.coherance, instance, matchdef, templatedef)
# stream.detrend('demean')
# stream.resample(samp_rate)
# stream.write('scripts/brightness_test_daylong.ms',format='MSEED')
stream=obsread('scripts/brightness_test_daylong.ms')
stream.trim(starttime=UTCDateTime('2011-09-04 17:05:00'),\
endtime=UTCDateTime('2011-09-04 17:15:00'))#, pad=True,\
# fill_value=0)
# for tr in stream:
# if tr.stats.station=='WVZ':
# stream.remove(tr)
stream.filter('bandpass',freqmin=4.0, freqmax=8.0)
# stream.trim(stream[0].stats.starttime+90, stream[0].stats.endtime)
stream.trim(stream[0].stats.starttime, stream[0].stats.endtime, pad=True, fill_value=0)
stream.plot(size=(800,600),equal_scale=False)
instance=0
# Cut the nodes...
cutnodes=[nodes[0]]+[nodes[116]]
cutlags=np.array([lags[:,0]]+[lags[:,116]]).T
detect_templates, detect_nodes=bright_lights.brightness(stations, \
nodes, lags, stream,
brightdef.threshold, brightdef.thresh_type,\
brightdef.coherance, instance, matchdef, templatedef,\
brightdef)
plotting.threeD_gridplot(detect_nodes)
# detect_templates, detect_nodes=bright_lights.brightness(stations, \
# cutnodes, cutlags, stream,
# brightdef.threshold, brightdef.thresh_type,\
# brightdef.coherance, instance, matchdef, templatedef)
def test_match_filter(samp_rate=20.0, debug=0):
"""
Function to test the capabilities of match_filter and just check that it\
is working! Uses synthetic templates and seeded, randomised data.
:type debug: int
:param debug: Debug level, higher the number the more output.
"""
from eqcorrscan.utils import pre_processing
from eqcorrscan.utils import EQcorrscan_plotting as plotting
from eqcorrscan.core import match_filter
from obspy import UTCDateTime
import string
# Generate a random dataset
templates, data, seeds = generate_synth_data(nsta=5,
ntemplates=2,
nseeds=50,
samp_rate=samp_rate,
t_length=6.0,
max_amp=5.0,
debug=debug)
# Notes to the user: If you use more templates you should ensure they are
# more different, e.g. set the data to have larger moveouts, otherwise
# similar templates will detect events seeded by another template.
# Test the pre_processing functions
for tr in data:
pre_processing.dayproc(tr=tr,
lowcut=2.0,
highcut=8.0,
filt_order=3,
samp_rate=20.0,
debug=0,
starttime=UTCDateTime(0))
if debug > 0:
data.plot()
# Filter the data and the templates
for template in templates:
pre_processing.shortproc(st=template,
lowcut=2.0,
highcut=8.0,
filt_order=3,
samp_rate=20.0)
if debug > 0:
template.plot()
template_names = list(string.ascii_lowercase)[0:len(templates)]
detections = match_filter.match_filter(template_names=template_names,
template_list=templates,
st=data,
threshold=8.0,
threshold_type='MAD',
trig_int=6.0,
plotvar=False,
plotdir='.',
cores=1)
# Compare the detections to the seeds
print('This test made ' + str(len(detections)) + ' detections')
ktrue = 0
kfalse = 0
for detection in detections:
print(detection.template_name)
i = template_names.index(detection.template_name)
t_seeds = seeds[i]
dtime_samples = int(
(detection.detect_time - UTCDateTime(0)) * samp_rate)
if dtime_samples in t_seeds['time']:
j = list(t_seeds['time']).index(dtime_samples)
print('Detection at SNR of: ' + str(t_seeds['SNR'][j]))
ktrue += 1
else:
min_diff = min(abs(t_seeds['time'] - dtime_samples))
if min_diff < 10:
# If there is a match within ten samples then its good enough
j = list(abs(t_seeds['time'] - dtime_samples)).index(min_diff)
print('Detection at SNR of: ' + str(t_seeds['SNR'][j]))
ktrue += 1
else:
print('Detection at sample: ' + str(dtime_samples) +
' does not match anything in seed times:')
kfalse += 1
print 'Minimum difference in samples is: ' + str(min_diff)
# Plot the detections
if debug > 3:
for i, template in enumerate(templates):
times = [
d.detect_time.datetime for d in detections
if d.template_name == template_names[i]
]
print(times)
plotting.detection_multiplot(data, template, times)
# Set an 'acceptable' ratio of positive to false detections
print(
str(ktrue) + ' true detections and ' + str(kfalse) +
' false detections')
if kfalse / ktrue < 0.25:
return True
else:
return False
def find_peaks2(arr, thresh, trig_int, debug=0, maxwidth=10,
starttime=False, samp_rate=1.0):
r"""Function to determine peaks in an array of data using scipy \
find_peaks_cwt, works fast in certain cases, but for match_filter cccsum \
peak finding, find_peaks2_short works better. Test it out and see which \
works best for your application.
:type arr: ndarray
:param arr: 1-D numpy array is required
:type thresh: float
:param thresh: The threshold below which will be considered noise and \
peaks will not be found in.
:type trig_int: int
:param trig_int: The minimum difference in samples between triggers, \
if multiple peaks within this window this code will find the highest.
:type debug: int
:param debug: Optional, debug level 0-5
:type maxwidth: int
:param maxwidth: Maximum peak width to look for in samples
:type starttime: osbpy.UTCDateTime
:param starttime: Starttime for plotting, only used if debug > 2.
:type samp_rate: float
:param samp_rate: Sampling rate in Hz, only used for plotting if debug > 2.
:return: peaks: Lists of tuples of peak values and locations.
"""
from scipy.signal import find_peaks_cwt
import numpy as np
from obspy import UTCDateTime
if not starttime:
starttime = UTCDateTime(0)
# Set everything below the threshold to zero
image = np.copy(arr)
image = np.abs(image)
image[image < thresh] = thresh
# We need to check if the number of samples in the image is prime, if it
# is this method will be really slow, so we add a pad to the end to make
# it not of prime length!
if is_prime(len(image)):
image = np.append(image, 0.0)
print 'Input array has a prime number of samples, appending a zero'
print len(image)
if len(image[image > thresh]) == 0:
print 'No values over threshold found'
return []
if debug > 0:
msg = ' '.join(['Found', str(len(image[image > thresh])),
'samples above the threshold'])
print msg
initial_peaks = []
peaks = []
# Find the peaks
print 'Finding peaks'
peakinds = find_peaks_cwt(image, np.arange(1, maxwidth))
initial_peaks = [(image[peakind], peakind) for peakind in peakinds]
# Sort initial peaks according to amplitude
print 'sorting peaks'
peaks_sort = sorted(initial_peaks, key=lambda amplitude: amplitude[0],
reverse=True)
if debug >= 4:
for peak in initial_peaks:
print peak
if initial_peaks:
peaks.append(peaks_sort[0]) # Definitely take the biggest peak
if debug > 3:
msg = ' '.join(['Added the biggest peak of', str(peaks[0][0]),
'at sample', str(peaks[0][1])])
print msg
if len(initial_peaks) > 1:
if debug > 3:
msg = ' '.join(['Multiple peaks found, checking them',
'now to see if they overlap'])
print msg
for next_peak in peaks_sort:
# i in xrange(1,len(peaks_sort)):
# Loop through the amplitude sorted peaks
# if the next highest amplitude peak is within trig_int of any
# peak already in peaks then we don't want it, else, add it
# next_peak = peaks_sort[i]
if debug > 3:
print next_peak
for peak in peaks:
add = False
# Use add as a switch for whether or not to append
# next peak to peaks, if once gone through all the peaks
# it is True, then we will add it, otherwise we won't!
if abs(next_peak[1] - peak[1]) < trig_int:
if debug > 3:
msg = ' '.join(['Difference in time is',
str(next_peak[1] - peak[1]), '\n'
'Which is less than',
str(trig_int)])
print msg
add = False
# Need to exit the loop here if false
break
else:
add = True
if add:
if debug > 3:
msg = ' '.join(['Adding peak of', str(next_peak[0]),
'at sample', str(next_peak[1])])
print msg
peaks.append(next_peak)
elif debug > 3:
msg = ' '.join(['I did not add peak of',
str(next_peak[0]), 'at sample',
str(next_peak[1])])
print msg
if debug >= 3:
from eqcorrscan.utils import EQcorrscan_plotting
_fname = ''.join(['peaks_',
starttime.datetime.strftime('%Y-%m-%d'),
'.pdf'])
print ' '.join(['Saving plot to', _fname])
EQcorrscan_plotting.peaks_plot(image, starttime, samp_rate, True,
peaks, _fname)
peaks = sorted(peaks, key=lambda time: time[1], reverse=False)
return peaks
else:
print 'No peaks for you!'
return peaks
def find_peaks_dep(arr, thresh, trig_int, debug=0, starttime=False,
samp_rate=1.0):
r"""Function to determine peaks in an array of data above a certain \
threshold.
Depreciated peak-finding routine, very slow, but accurate. If all else \
fails this one should work.
:type arr: ndarray
:param arr: 1-D numpy array is required
:type thresh: float
:param thresh: The threshold below which will be considered noise and \
peaks will not be found in.
:type trig_int: int
:param trig_int: The minimum difference in samples between triggers,\
if multiple peaks within this window this code will find the highest.
:type starttime: osbpy.UTCDateTime
:param starttime: Starttime for plotting, only used if debug > 2.
:type samp_rate: float
:param samp_rate: Sampling rate in Hz, only used for plotting if debug > 2.
:return: peaks: Lists of tuples of peak values and locations.
"""
import numpy as np
from obspy import UTCDateTime
if not starttime:
starttime = UTCDateTime(0)
# Perform some checks
if trig_int < 3:
import sys
print 'Trigger interval must be greater than 2 samples to find maxima'
sys.exit()
# from joblib import Parallel, delayed
# Will find peaks in the absolute then transfer these to the true values
sig = np.abs(arr) - thresh
true_peaks = []
for i in xrange(int(trig_int), int(len(sig) - trig_int), int(trig_int)):
window = sig[i - trig_int: i + trig_int]
# Define a moving window containing data from +/- the trigger iterval
peaks = []
locs = []
for j in xrange(1, len(window) - 1):
# Find all turning points within the window
if window[j] > 0.0 and window[j] > window[j+1] and\
window[j] > window[j - 1]:
peaks.append(window[j])
locs.append(i - trig_int + j)
# Find maximum peak in window
if peaks:
true_peaks.append((np.max(np.array(peaks)),
locs[np.argmax(np.array(peaks))]))
# Get unique values
peaks = sorted(list(set(true_peaks)), key=lambda loc: loc[1])
# Find highest peak in peaks within trig_int of each other
for i in xrange(1, len(peaks) - 1):
if peaks[i + 1][1]-peaks[i][1] < trig_int:
if peaks[i][0] < peaks[i + 1][0]:
peaks[i] = peaks[i + 1]
else:
peaks[i + 1] = peaks[i]
elif peaks[i][1] - peaks[i - 1][1] < trig_int:
if peaks[i][0] < peaks[i - 1][0]:
peaks[i] = peaks[i - 1]
else:
peaks[i - 1] = peaks[i]
peaks = sorted(list(set(peaks)), key=lambda loc: loc[1])
if debug >= 3:
from eqcorrscan.utils import EQcorrscan_plotting
_fname = ''.join(['peaks_',
starttime.datetime.strftime('%Y-%m-%d'),
'.pdf'])
EQcorrscan_plotting.peaks_plot(arr, starttime, samp_rate, True, peaks,
_fname)
return peaks
def find_peaks2_short(arr, thresh, trig_int, debug=0, starttime=False,
samp_rate=1.0):
r"""Function to determine peaks in an array of data above a certain \
threshold. Uses a mask to remove data below threshold and finds peaks in \
what is left.
:type arr: ndarray
:param arr: 1-D numpy array is required
:type thresh: float
:param thresh: The threshold below which will be considered noise and \
peaks will not be found in.
:type trig_int: int
:param trig_int: The minimum difference in samples between triggers,\
if multiple peaks within this window this code will find the highest.
:type debug: int
:param debug: Optional, debug level 0-5
:type starttime: osbpy.UTCDateTime
:param starttime: Starttime for plotting, only used if debug > 2.
:type samp_rate: float
:param samp_rate: Sampling rate in Hz, only used for plotting if debug > 2.
:return: peaks: Lists of tuples of peak values and locations.
"""
from scipy import ndimage
import numpy as np
from obspy import UTCDateTime
if not starttime:
starttime = UTCDateTime(0)
# Set everything below the threshold to zero
image = np.copy(arr)
image = np.abs(image)
image[image < thresh] = 0
if len(image[image > thresh]) == 0:
print 'No values over threshold found'
return []
if debug > 0:
print ' '.join(['Found', str(len(image[image > thresh])),
'samples above the threshold'])
initial_peaks = []
peaks = []
# Find the peaks
labeled_image, number_of_objects = ndimage.label(image)
peak_slices = ndimage.find_objects(labeled_image)
for peak_slice in peak_slices:
# print 'Width of peak='+str(peak_slice[0].stop-peak_slice[0].start)
window = arr[peak_slice[0].start: peak_slice[0].stop]
initial_peaks.append((max(window),
peak_slice[0].start + np.argmax(window)))
# Sort initial peaks according to amplitude
peaks_sort = sorted(initial_peaks, key=lambda amplitude: amplitude[0],
reverse=True)
# Debugging
if debug >= 4:
for peak in initial_peaks:
print peak
if initial_peaks:
peaks.append(peaks_sort[0]) # Definitely take the biggest peak
if debug > 3:
print ' '.join(['Added the biggest peak of', str(peaks[0][0]),
'at sample', str(peaks[0][1])])
if len(initial_peaks) > 1:
if debug > 3:
msg = ' '.join(['Multiple peaks found, checking',
'them now to see if they overlap'])
print msg
for next_peak in peaks_sort:
# i in xrange(1,len(peaks_sort)):
# Loop through the amplitude sorted peaks
# if the next highest amplitude peak is within trig_int of any
# peak already in peaks then we don't want it, else, add it
# next_peak=peaks_sort[i]
if debug > 3:
print next_peak
for peak in peaks:
add = False
# Use add as a switch for whether or not to append
# next peak to peaks, if once gone through all the peaks
# it is True, then we will add it, otherwise we won't!
if abs(next_peak[1]-peak[1]) < trig_int:
if debug > 3:
msg = ' '.join(['Difference in time is',
str(next_peak[1]-peak[1]), '\n',
'Which is less than',
str(trig_int)])
print msg
add = False
# Need to exit the loop here if false
break
else:
add = True
if add:
if debug > 3:
msg = ' '.join(['Adding peak of', str(next_peak[0]),
'at sample', str(next_peak[1])])
print msg
peaks.append(next_peak)
elif debug > 3:
msg = ' '.join(['I did not add peak of', str(next_peak[0]),
'at sample', str(next_peak[1])])
print msg
if debug >= 3:
from eqcorrscan.utils import EQcorrscan_plotting
_fname = ''.join(['peaks_',
starttime.datetime.strftime('%Y-%m-%d'),
'.pdf'])
EQcorrscan_plotting.peaks_plot(image, starttime, samp_rate, True,
peaks, _fname)
peaks = sorted(peaks, key=lambda time: time[1], reverse=False)
return peaks
else:
print 'No peaks for you!'
return peaks
def match_filter(template_names, template_list, st, threshold,
threshold_type, trig_int, plotvar, plotdir='.', cores=1,
tempdir=False, debug=0, plot_format='jpg'):
r"""Over-arching code to run the correlations of given templates with a\
day of seismic data and output the detections based on a given threshold.
:type template_names: list
:param template_names: List of template names in the same order as\
template_list
:type template_list: list :class: 'obspy.Stream'
:param template_list: A list of templates of which each template is a\
Stream of obspy traces containing seismic data and header information.
:type st: :class: 'obspy.Stream'
:param st: An obspy.Stream object containing all the data available and\
required for the correlations with templates given. For efficiency\
this should contain no excess traces which are not in one or more of\
the templates. This will now remove excess traces internally, but\
will copy the stream and work on the copy, leaving your input stream\
untouched.
:type threshold: float
:param threshold: A threshold value set based on the threshold_type
:type threshold_type: str
:param threshold_type: The type of threshold to be used, can be MAD,\
absolute or av_chan_corr. MAD threshold is calculated as the\
threshold*(median(abs(cccsum))) where cccsum is the cross-correlation\
sum for a given template. absolute threhsold is a true absolute\
threshold based on the cccsum value av_chan_corr is based on the mean\
values of single-channel cross-correlations assuming all data are\
present as required for the template, \
e.g. av_chan_corr_thresh=threshold*(cccsum/len(template)) where\
template is a single template from the input and the length is the\
number of channels within this template.
:type trig_int: float
:param trig_int: Minimum gap between detections in seconds.
:type plotvar: bool
:param plotvar: Turn plotting on or off
:type plotdir: str
:param plotdir: Path to plotting folder, plots will be output here,\
defaults to run location.
:type tempdir: String or False
:param tempdir: Directory to put temporary files, or False
:type cores: int
:param cores: Number of cores to use
:type debug: int
:param debug: Debug output level, the bigger the number, the more the\
output.
:return: :class: 'DETECTIONS' detections for each channel formatted as\
:class: 'obspy.UTCDateTime' objects.
.. rubric:: Note
Plotting within the match-filter routine uses the Agg backend with\
interactive plotting turned off. This is because the function is\
designed to work in bulk. If you wish to turn interactive plotting on\
you must import matplotlib in your script first, when you them import\
match_filter you will get the warning that this call to matplotlib has\
no effect, which will mean that match_filter has not changed the\
plotting behaviour.
"""
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.ioff()
import copy
from eqcorrscan.utils import EQcorrscan_plotting
from eqcorrscan.utils import findpeaks
from obspy import Trace
import time
# Copy the stream here because we will f**k about with it
stream = st.copy()
templates = copy.deepcopy(template_list)
# Debug option to confirm that the channel names match those in the
# templates
if debug >= 2:
template_stachan = []
data_stachan = []
for template in templates:
for tr in template:
template_stachan.append(tr.stats.station + '.' +
tr.stats.channel)
for tr in stream:
data_stachan.append(tr.stats.station + '.' + tr.stats.channel)
template_stachan = list(set(template_stachan))
data_stachan = list(set(data_stachan))
if debug >= 3:
print 'I have template info for these stations:'
print template_stachan
print 'I have daylong data for these stations:'
print data_stachan
# Perform a check that the daylong vectors are daylong
for tr in stream:
if not tr.stats.sampling_rate * 86400 == tr.stats.npts:
msg = ' '.join(['Data are not daylong for', tr.stats.station,
tr.stats.channel])
raise ValueError(msg)
# Call the _template_loop function to do all the correlation work
outtic = time.clock()
# Edit here from previous, stable, but slow match_filter
# Would be worth testing without an if statement, but with every station in
# the possible template stations having data, but for those without real
# data make the data NaN to return NaN ccc_sum
# Note: this works
if debug >= 2:
print 'Ensuring all template channels have matches in daylong data'
template_stachan = []
for template in templates:
for tr in template:
template_stachan += [(tr.stats.station, tr.stats.channel)]
template_stachan = list(set(template_stachan))
# Copy this here to keep it safe
for stachan in template_stachan:
if not stream.select(station=stachan[0], channel=stachan[1]):
# Remove template traces rather than adding NaN data
for template in templates:
if template.select(station=stachan[0], channel=stachan[1]):
for tr in template.select(station=stachan[0],
channel=stachan[1]):
template.remove(tr)
# Remove un-needed channels
for tr in stream:
if not (tr.stats.station, tr.stats.channel) in template_stachan:
stream.remove(tr)
# Also pad out templates to have all channels
for template in templates:
for stachan in template_stachan:
if not template.select(station=stachan[0], channel=stachan[1]):
nulltrace = Trace()
nulltrace.stats.station = stachan[0]
nulltrace.stats.channel = stachan[1]
nulltrace.stats.sampling_rate = template[0].stats.sampling_rate
nulltrace.stats.starttime = template[0].stats.starttime
nulltrace.data = np.array([np.NaN] * len(template[0].data),
dtype=np.float32)
template += nulltrace
if debug >= 2:
print 'Starting the correlation run for this day'
[cccsums, no_chans] = _channel_loop(templates, stream, cores, debug)
if len(cccsums[0]) == 0:
raise ValueError('Correlation has not run, zero length cccsum')
outtoc = time.clock()
print ' '.join(['Looping over templates and streams took:',
str(outtoc - outtic), 's'])
if debug >= 2:
print ' '.join(['The shape of the returned cccsums is:',
str(np.shape(cccsums))])
print ' '.join(['This is from', str(len(templates)), 'templates'])
print ' '.join(['Correlated with', str(len(stream)),
'channels of data'])
detections = []
for i, cccsum in enumerate(cccsums):
template = templates[i]
if threshold_type == 'MAD':
rawthresh = threshold * np.median(np.abs(cccsum))
elif threshold_type == 'absolute':
rawthresh = threshold
elif threshold == 'av_chan_corr':
rawthresh = threshold * (cccsum / len(template))
else:
print 'You have not selected the correct threshold type, I will' +\
'use MAD as I like it'
rawthresh = threshold * np.mean(np.abs(cccsum))
# Findpeaks returns a list of tuples in the form [(cccsum, sample)]
print ' '.join(['Threshold is set at:', str(rawthresh)])
print ' '.join(['Max of data is:', str(max(cccsum))])
print ' '.join(['Mean of data is:', str(np.mean(cccsum))])
if np.abs(np.mean(cccsum)) > 0.05:
warnings.warn('Mean is not zero! Check this!')
# Set up a trace object for the cccsum as this is easier to plot and
# maintins timing
if plotvar:
stream_plot = copy.deepcopy(stream[0])
# Downsample for plotting
stream_plot.decimate(int(stream[0].stats.sampling_rate / 10))
cccsum_plot = Trace(cccsum)
cccsum_plot.stats.sampling_rate = stream[0].stats.sampling_rate
# Resample here to maintain shape better
cccsum_hist = cccsum_plot.copy()
cccsum_hist = cccsum_hist.decimate(int(stream[0].stats.sampling_rate /
10)).data
cccsum_plot = EQcorrscan_plotting.chunk_data(cccsum_plot, 10,
'Maxabs').data
# Enforce same length
stream_plot.data = stream_plot.data[0:len(cccsum_plot)]
cccsum_plot = cccsum_plot[0:len(stream_plot.data)]
cccsum_hist = cccsum_hist[0:len(stream_plot.data)]
EQcorrscan_plotting.triple_plot(cccsum_plot, cccsum_hist,
stream_plot, rawthresh, True,
plotdir + '/cccsum_plot_' +
template_names[i] + '_' +
stream[0].stats.starttime.datetime.strftime('%Y-%m-%d') +
'.' + plot_format)
if debug >= 4:
print ' '.join(['Saved the cccsum to:', template_names[i],
stream[0].stats.starttime.datetime.strftime('%Y%j')])
np.save(template_names[i] +
stream[0].stats.starttime.datetime.strftime('%Y%j'),
cccsum)
tic = time.clock()
if debug >= 4:
np.save('cccsum_' + str(i) + '.npy', cccsum)
if debug >= 3 and max(cccsum) > rawthresh:
peaks = findpeaks.find_peaks2_short(cccsum, rawthresh,
trig_int * stream[0].stats.sampling_rate,
debug,
stream[0].stats.starttime,
stream[0].stats.sampling_rate)
elif max(cccsum) > rawthresh:
peaks = findpeaks.find_peaks2_short(cccsum, rawthresh,
trig_int * stream[0].stats.sampling_rate,
debug)
else:
print 'No peaks found above threshold'
peaks = False
toc = time.clock()
if debug >= 1:
print ' '.join(['Finding peaks took:', str(toc - tic), 's'])
if peaks:
for peak in peaks:
detecttime = stream[0].stats.starttime +\
peak[1] / stream[0].stats.sampling_rate
detections.append(DETECTION(template_names[i],
detecttime,
no_chans[i], peak[0], rawthresh,
'corr'))
del stream, templates
return detections
def find_peaks2_short(arr,
thresh,
trig_int,
debug=0,
starttime=False,
samp_rate=1.0):
r"""Function to determine peaks in an array of data above a certain \
threshold. Uses a mask to remove data below threshold and finds peaks in \
what is left.
:type arr: ndarray
:param arr: 1-D numpy array is required
:type thresh: float
:param thresh: The threshold below which will be considered noise and \
peaks will not be found in.
:type trig_int: int
:param trig_int: The minimum difference in samples between triggers,\
if multiple peaks within this window this code will find the highest.
:type debug: int
:param debug: Optional, debug level 0-5
:type starttime: osbpy.UTCDateTime
:param starttime: Starttime for plotting, only used if debug > 2.
:type samp_rate: float
:param samp_rate: Sampling rate in Hz, only used for plotting if debug > 2.
:return: peaks: Lists of tuples of peak values and locations.
"""
from scipy import ndimage
import numpy as np
from obspy import UTCDateTime
if not starttime:
starttime = UTCDateTime(0)
# Set everything below the threshold to zero
image = np.copy(arr)
image = np.abs(image)
image[image < thresh] = 0
if len(image[image > thresh]) == 0:
print('No values over threshold found')
return []
if debug > 0:
print(' '.join([
'Found',
str(len(image[image > thresh])), 'samples above the threshold'
]))
initial_peaks = []
peaks = []
# Find the peaks
labeled_image, number_of_objects = ndimage.label(image)
peak_slices = ndimage.find_objects(labeled_image)
for peak_slice in peak_slices:
# print('Width of peak='+str(peak_slice[0].stop-peak_slice[0].start)
window = arr[peak_slice[0].start:peak_slice[0].stop]
initial_peaks.append(
(max(window), peak_slice[0].start + np.argmax(window)))
# Sort initial peaks according to amplitude
peaks_sort = sorted(initial_peaks,
key=lambda amplitude: amplitude[0],
reverse=True)
# Debugging
if debug >= 4:
for peak in initial_peaks:
print(peak)
if initial_peaks:
peaks.append(peaks_sort[0]) # Definitely take the biggest peak
if debug > 3:
print(' '.join([
'Added the biggest peak of',
str(peaks[0][0]), 'at sample',
str(peaks[0][1])
]))
if len(initial_peaks) > 1:
if debug > 3:
msg = ' '.join([
'Multiple peaks found, checking',
'them now to see if they overlap'
])
print(msg)
for next_peak in peaks_sort:
# i in xrange(1,len(peaks_sort)):
# Loop through the amplitude sorted peaks
# if the next highest amplitude peak is within trig_int of any
# peak already in peaks then we don't want it, else, add it
# next_peak=peaks_sort[i]
if debug > 3:
print(next_peak)
for peak in peaks:
add = False
# Use add as a switch for whether or not to append
# next peak to peaks, if once gone through all the peaks
# it is True, then we will add it, otherwise we won't!
if abs(next_peak[1] - peak[1]) < trig_int:
if debug > 3:
msg = ' '.join([
'Difference in time is',
str(next_peak[1] - peak[1]), '\n',
'Which is less than',
str(trig_int)
])
print(msg)
add = False
# Need to exit the loop here if false
break
else:
add = True
if add:
if debug > 3:
msg = ' '.join([
'Adding peak of',
str(next_peak[0]), 'at sample',
str(next_peak[1])
])
print(msg)
peaks.append(next_peak)
elif debug > 3:
msg = ' '.join([
'I did not add peak of',
str(next_peak[0]), 'at sample',
str(next_peak[1])
])
print(msg)
if debug >= 3:
from eqcorrscan.utils import EQcorrscan_plotting
_fname = ''.join(
['peaks_',
starttime.datetime.strftime('%Y-%m-%d'), '.pdf'])
EQcorrscan_plotting.peaks_plot(image, starttime, samp_rate, True,
peaks, _fname)
peaks = sorted(peaks, key=lambda time: time[1], reverse=False)
return peaks
else:
print('No peaks for you!')
return peaks
def brightness(stations,
nodes,
lags,
stream,
threshold,
thresh_type,
template_length,
template_saveloc,
coherence_thresh,
coherence_stations=['all'],
coherence_clip=False,
gap=2.0,
clip_level=100,
instance=0,
pre_pick=0.2,
plotsave=True,
cores=1):
r"""Function to calculate the brightness function in terms of energy for \
a day of data over the entire network for a given grid of nodes.
Note data in stream must be all of the same length and have the same
sampling rates.
:type stations: list
:param stations: List of station names from in the form where stations[i] \
refers to nodes[i][:] and lags[i][:]
:type nodes: list, tuple
:param nodes: List of node points where nodes[i] referes to stations[i] \
and nodes[:][:][0] is latitude in degrees, nodes[:][:][1] is \
longitude in degrees, nodes[:][:][2] is depth in km.
:type lags: :class: 'numpy.array'
:param lags: Array of arrays where lags[i][:] refers to stations[i]. \
lags[i][j] should be the delay to the nodes[i][j] for stations[i] in \
seconds.
:type stream: :class: `obspy.Stream`
:param data: Data through which to look for detections.
:type threshold: float
:param threshold: Threshold value for detection of template within the \
brightness function
:type thresh_type: str
:param thresh_type: Either MAD or abs where MAD is the Median Absolute \
Deviation and abs is an absoulte brightness.
:type template_length: float
:param template_length: Length of template to extract in seconds
:type template_saveloc: str
:param template_saveloc: Path of where to save the templates.
:type coherence_thresh: tuple of floats
:param coherence_thresh: Threshold for removing incoherant peaks in the \
network response, those below this will not be used as templates. \
Must be in the form of (a,b) where the coherence is given by: \
a-kchan/b where kchan is the number of channels used to compute \
the coherence
:type coherence_stations: list
:param coherence_stations: List of stations to use in the coherance \
thresholding - defaults to 'all' which uses all the stations.
:type coherence_clip: float
:param coherence_clip: tuple
:type coherence_clip: Start and end in seconds of data to window around, \
defaults to False, which uses all the data given.
:type pre_pick: float
:param pre_pick: Seconds before the detection time to include in template
:type plotsave: bool
:param plotsave: Save or show plots, if False will try and show the plots \
on screen - as this is designed for bulk use this is set to \
True to save any plots rather than show them if you create \
them - changes the backend of matplotlib, so if is set to \
False you will see NO PLOTS!
:type cores: int
:param core: Number of cores to use, defaults to 1.
:type clip_level: float
:param clip_level: Multiplier applied to the mean deviation of the energy \
as an upper limit, used to remove spikes (earthquakes, \
lightning, electircal spikes) from the energy stack.
:type gap: float
:param gap: Minimum inter-event time in seconds for detections
:return: list of templates as :class: `obspy.Stream` objects
"""
from eqcorrscan.core.template_gen import _template_gen
if plotsave:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.ioff()
# from joblib import Parallel, delayed
from multiprocessing import Pool, cpu_count
from copy import deepcopy
from obspy import read as obsread
from obspy.core.event import Catalog, Event, Pick, WaveformStreamID, Origin
from obspy.core.event import EventDescription, CreationInfo, Comment
import obspy.Stream
import matplotlib.pyplot as plt
from eqcorrscan.utils import EQcorrscan_plotting as plotting
# Check that we actually have the correct stations
realstations = []
for station in stations:
st = stream.select(station=station)
if st:
realstations += station
del st
stream_copy = stream.copy()
# Force convert to int16
for tr in stream_copy:
# int16 max range is +/- 32767
if max(abs(tr.data)) > 32767:
tr.data = 32767 * (tr.data / max(abs(tr.data)))
# Make sure that the data aren't clipped it they are high gain
# scale the data
tr.data = tr.data.astype(np.int16)
# The internal _node_loop converts energy to int16 too to converse memory,
# to do this it forces the maximum of a single energy trace to be 500 and
# normalises to this level - this only works for fewer than 65 channels of
# data
if len(stream_copy) > 130:
raise OverflowError('Too many streams, either re-code and cope with' +
'either more memory usage, or less precision, or' +
'reduce data volume')
detections = []
detect_lags = []
parallel = True
plotvar = True
mem_issue = False
# Loop through each node in the input
# Linear run
print('Computing the energy stacks')
if not parallel:
for i in range(0, len(nodes)):
print(i)
if not mem_issue:
j, a = _node_loop(stations, lags[:, i], stream, plot=True)
if 'energy' not in locals():
energy = a
else:
energy = np.concatenate((energy, a), axis=0)
print('energy: ' + str(np.shape(energy)))
else:
j, filename = _node_loop(stations, lags[:, i], stream, i,
mem_issue)
energy = np.array(energy)
print(np.shape(energy))
else:
# Parallel run
num_cores = cores
if num_cores > len(nodes):
num_cores = len(nodes)
if num_cores > cpu_count():
num_cores = cpu_count()
pool = Pool(processes=num_cores)
results = [
pool.apply_async(_node_loop,
args=(stations, lags[:, i], stream, i, clip_level,
mem_issue, instance))
for i in range(len(nodes))
]
pool.close()
if not mem_issue:
print('Computing the cumulative network response from memory')
energy = [p.get() for p in results]
pool.join()
energy.sort(key=lambda tup: tup[0])
energy = [node[1] for node in energy]
energy = np.concatenate(energy, axis=0)
print(energy.shape)
else:
pool.join()
# Now compute the cumulative network response and then detect possible
# events
if not mem_issue:
print(energy.shape)
indeces = np.argmax(energy, axis=0) # Indeces of maximum energy
print(indeces.shape)
cum_net_resp = np.array([np.nan] * len(indeces))
cum_net_resp[0] = energy[indeces[0]][0]
peak_nodes = [nodes[indeces[0]]]
for i in range(1, len(indeces)):
cum_net_resp[i] = energy[indeces[i]][i]
peak_nodes.append(nodes[indeces[i]])
del energy, indeces
else:
print('Reading the temp files and computing network response')
node_splits = len(nodes) // num_cores
indeces = [range(node_splits)]
for i in range(1, num_cores - 1):
indeces.append(range(node_splits * i, node_splits * (i + 1)))
indeces.append(range(node_splits * (i + 1), len(nodes)))
pool = Pool(processes=num_cores)
results = [
pool.apply_async(_cum_net_resp, args=(indeces[i], instance))
for i in range(num_cores)
]
pool.close()
results = [p.get() for p in results]
pool.join()
responses = [result[0] for result in results]
print(np.shape(responses))
node_indeces = [result[1] for result in results]
cum_net_resp = np.array(responses)
indeces = np.argmax(cum_net_resp, axis=0)
print(indeces.shape)
print(cum_net_resp.shape)
cum_net_resp = np.array(
[cum_net_resp[indeces[i]][i] for i in range(len(indeces))])
peak_nodes = [
nodes[node_indeces[indeces[i]][i]] for i in range(len(indeces))
]
del indeces, node_indeces
if plotvar:
cum_net_trace = deepcopy(stream[0])
cum_net_trace.data = cum_net_resp
cum_net_trace.stats.station = 'NR'
cum_net_trace.stats.channel = ''
cum_net_trace.stats.network = 'Z'
cum_net_trace.stats.location = ''
cum_net_trace.stats.starttime = stream[0].stats.starttime
cum_net_trace = obspy.Stream(cum_net_trace)
cum_net_trace += stream.select(channel='*N')
cum_net_trace += stream.select(channel='*1')
cum_net_trace.sort(['network', 'station', 'channel'])
# np.save('cum_net_resp.npy',cum_net_resp)
# cum_net_trace.plot(size=(800,600), equal_scale=False,\
# outfile='NR_timeseries.eps')
# Find detection within this network response
print('Finding detections in the cumulatve network response')
detections = _find_detections(cum_net_resp, peak_nodes, threshold,
thresh_type, stream[0].stats.sampling_rate,
realstations, gap)
del cum_net_resp
templates = []
nodesout = []
good_detections = []
if detections:
print('Converting detections in to templates')
# Generate a catalog of detections
detections_cat = Catalog()
for j, detection in enumerate(detections):
print('Converting for detection ' + str(j) + ' of ' +
str(len(detections)))
# Create an event for each detection
event = Event()
# Set up some header info for the event
event.event_descriptions.append(EventDescription())
event.event_descriptions[0].text = 'Brightness detection'
event.creation_info = CreationInfo(agency_id='EQcorrscan')
copy_of_stream = deepcopy(stream_copy)
# Convert detections to obspy.core.event type -
# name of detection template is the node.
node = (detection.template_name.split('_')[0],
detection.template_name.split('_')[1],
detection.template_name.split('_')[2])
print(node)
# Look up node in nodes and find the associated lags
index = nodes.index(node)
detect_lags = lags[:, index]
ksta = Comment(text='Number of stations=' + len(detect_lags))
event.origins.append(Origin())
event.origins[0].comments.append(ksta)
event.origins[0].time = copy_of_stream[0].stats.starttime +\
detect_lags[0] + detection.detect_time
event.origins[0].latitude = node[0]
event.origins[0].longitude = node[1]
event.origins[0].depth = node[2]
for i, detect_lag in enumerate(detect_lags):
station = stations[i]
st = copy_of_stream.select(station=station)
if len(st) != 0:
for tr in st:
_waveform_id = WaveformStreamID(
station_code=tr.stats.station,
channel_code=tr.stats.channel,
network_code='NA')
event.picks.append(
Pick(waveform_id=_waveform_id,
time=tr.stats.starttime + detect_lag +
detection.detect_time + pre_pick,
onset='emergent',
evalutation_mode='automatic'))
print('Generating template for detection: ' + str(j))
template = (_template_gen(event.picks, copy_of_stream,
template_length, 'all'))
template_name = template_saveloc + '/' +\
str(template[0].stats.starttime) + '.ms'
# In the interests of RAM conservation we write then read
# Check coherancy here!
temp_coher, kchan = coherence(template, coherence_stations,
coherence_clip)
coh_thresh = float(coherence_thresh[0]) - kchan / \
float(coherence_thresh[1])
if temp_coher > coh_thresh:
template.write(template_name, format="MSEED")
print('Written template as: ' + template_name)
print('---------------------------------coherence LEVEL: ' +
str(temp_coher))
coherant = True
else:
print('Template was incoherant, coherence level: ' +
str(temp_coher))
coherant = False
del copy_of_stream, tr, template
if coherant:
templates.append(obsread(template_name))
nodesout += [node]
good_detections.append(detection)
else:
print('No template for you')
if plotvar:
all_detections = [(cum_net_trace[-1].stats.starttime +
detection.detect_time).datetime
for detection in detections]
good_detections = [(cum_net_trace[-1].stats.starttime +
detection.detect_time).datetime
for detection in good_detections]
if not plotsave:
plotting.NR_plot(cum_net_trace[0:-1],
obspy.Stream(cum_net_trace[-1]),
detections=good_detections,
size=(18.5, 10),
title='Network response')
# cum_net_trace.plot(size=(800,600), equal_scale=False)
else:
savefile = 'plots/' +\
cum_net_trace[0].stats.starttime.datetime.strftime('%Y%m%d') +\
'_NR_timeseries.pdf'
plotting.NR_plot(cum_net_trace[0:-1],
obspy.Stream(cum_net_trace[-1]),
detections=good_detections,
size=(18.5, 10),
save=savefile,
title='Network response')
nodesout = list(set(nodesout))
return templates, nodesout
# Now lets try and work out how many unique events we have just to compare
# with the GeoNet catalog of 20 events on this day in this sequence
for master in detections:
keep = True
for slave in detections:
if not master == slave and\
abs(master.detect_time - slave.detect_time) <= 6.0:
# If the events are within 6s of each other then test which
# was the 'best' match, strongest detection
if not master.detect_val > slave.detect_val:
keep = False
break
if keep:
unique_detections.append(master)
print('We made a total of ' + str(len(unique_detections)) + ' detections')
for detection in unique_detections:
print('Detection at :' + str(detection.detect_time) + ' for template ' +
detection.template_name + ' with a cross-correlation sum of: ' +
str(detection.detect_val))
# We can plot these too
stplot = st.copy()
template = templates[template_names.index(detection.template_name)]
lags = sorted([tr.stats.starttime for tr in template])
maxlag = lags[-1] - lags[0]
stplot.trim(starttime=detection.detect_time - 10,
endtime=detection.detect_time + maxlag + 10)
plotting.detection_multiplot(stplot, template,
[detection.detect_time.datetime])
