def get_or_attach_response(self,
trace,
dataless_inventories=(),
xml_inventories=()):
"""
Returns or attach instrumental response, from dataless seed inventories
(as returned by psstation.get_dataless_inventories()) and/or StationXML
inventories (as returned by psstation.get_stationxml_inventories()).
If a response if found in a dataless inventory, then a dict of poles
and zeros is returned. If a response is found in a StationXML
inventory, then it is directly attached to the trace and nothing is
returned.
Raises CannotPreprocess exception if no instrumental response is found.
@type trace: L{Trace}
@param dataless_inventories: inventories from dataless seed files
(as returned by
psstation.get_dataless_inventories())
@type dataless_inventories: list of L{obspy.xseed.parser.Parser}
@param xml_inventories: inventories from StationXML files
(as returned by
psstation.get_stationxml_inventories())
@type xml_inventories: list of L{obspy.station.inventory.Inventory}
"""
t1 = dt.datetime.now()
# looking for instrument response...
try:
# ...first in dataless seed inventories
paz = psstation.get_paz(channelid=trace.id,
t=trace.stats.starttime,
inventories=dataless_inventories)
return paz
except pserrors.NoPAZFound:
# ... then in dataless seed inventories, replacing 'BHZ' with 'HHZ'
# in trace's id (trick to make code work with Diogo's data)
try:
paz = psstation.get_paz(channelid=trace.id.replace(
'BHZ', 'HHZ'),
t=trace.stats.starttime,
inventories=dataless_inventories)
return paz
except pserrors.NoPAZFound:
# ...finally in StationXML inventories
try:
trace.attach_response(inventories=xml_inventories)
except:
# no response found!
raise pserrors.CannotPreprocess("No response found")
delta = (dt.datetime.now() - t1).total_seconds()
print "\nProcessed response attachment in {:.1f} seconds".format(delta)
def time_norm(self, trace, trcopy):
if self.onebit_norm:
# one-bit normalization
trace.data = np.sign(trace.data)
else:
# normalization of the signal by the running mean
# in the earthquake frequency band
trcopy.filter(type="bandpass",
freqmin=self.freqmin_earthquake,
freqmax=self.freqmax_earthquake,
corners=self.corners,
zerophase=self.zerophase)
# decimating trace
psutils.resample(trcopy, self.period_resample)
# Time-normalization weights from smoothed abs(data)
# Note that trace's data can be a masked array
halfwindow = int(
round(self.window_time * trcopy.stats.sampling_rate / 2))
mask = ~trcopy.data.mask if np.ma.isMA(trcopy.data) else None
tnorm_w = psutils.moving_avg(np.abs(trcopy.data),
halfwindow=halfwindow,
mask=mask)
if np.ma.isMA(trcopy.data):
# turning time-normalization weights into a masked array
s = "[warning: {}.{} trace's data is a masked array]"
print s.format(trace.stats.network, trace.stats.station),
tnorm_w = np.ma.masked_array(tnorm_w, trcopy.data.mask)
if np.any((tnorm_w == 0.0) | np.isnan(tnorm_w)):
# illegal normalizing value -> skipping trace
raise pserrors.CannotPreprocess("Zero or NaN normalization weight")
# time-normalization
trace.data /= tnorm_w
return trace
def get_merged_trace(self,
station,
date,
xcorr_interval,
skiplocs=CROSSCORR_SKIPLOCS,
minfill=MINFILL,
verbose=False):
"""
Returns one trace extracted from selected station, at selected date
(+/- 1 hour on each side to avoid edge effects during subsequent
processing).
for 45 minute xcorr interval, change edge effects by 1 minute!
Traces whose location belongs to *skiplocs* are discarded, then
if several locations remain, only the first is kept. Finally,
if several traces (with the same location) remain, they are
merged, WITH GAPS FILLED USING LINEAR INTERPOLATION.
Raises CannotPreprocess exception if:
- no trace remain after discarded the unwanted locations
- data fill is < *minfill*
@type station: L{psstation.Station}
@type date: L{datetime.date}
@param skiplocs: list of locations to discard in station's data
@type skiplocs: iterable
@param minfill: minimum data fill to keep trace
@rtype: L{Trace}
"""
#calculate edge addition and subtraction as 1/24th of the overall time interval
#
startminutes = (xcorr_interval / 24.0)
endminutes = xcorr_interval + startminutes
# getting station's stream at selected date
# (+/- one hour to avoid edge effects when removing response)
t0 = UTCDateTime(date) # date at time 00h00m00s
path_start = t0 - dt.timedelta(minutes=startminutes)
path_end = t0 + dt.timedelta(minutes=endminutes)
#station_path_old = station.getpath(date, MSEED_DIR)
station_path_SQL = station.getpath(t0, t0+dt.timedelta\
(minutes=xcorr_interval))
#print "station old path: ", station_path_old
#print "station SQl path: ", station_path_SQL
#print
#print "SQL path: ", station_path_SQL
try:
st = read(pathname_or_url=station_path_SQL,
starttime=path_start,
endtime=path_end)
except:
st = read_ref(station_path_SQL)
st = st.trim(starttime=path_start, endtime=path_end)
# check traces
for tr in st:
path_info = station_path_SQL.split('/')
alt_station = path_info[-3]
stats = tr.stats
network = stats.network
station = stats.station
channel = stats.channel
if station == '' and len(alt_station) == 4:
station = path_info[-3]
if network == '':
network = 'XX'
if len(channel) == 1:
channel = 'DH' + channel
tr.stats.channel = channel
tr.stats.station = station
tr.stats.network = network
# MAKE THIS AN OPTION IN THE CONFIGURATION FILES!
st = st.select(component='Z')
#print "st: ", st
#st = read(pathname_or_url=station.getpath(date),
# starttime=t0 - dt.timedelta(hours=1),
# endtime=t0 + dt.timedelta(days=1, hours=1))
# removing traces of stream from locations to skip
for tr in [tr for tr in st if tr.stats.location in skiplocs]:
st.remove(tr)
if not st.traces:
# no remaining trace!
raise pserrors.CannotPreprocess("No trace")
# if more than one location, we retain only the first one
if len(set(tr.id for tr in st)) > 1:
select_loc = sorted(set(tr.stats.location for tr in st))[0]
for tr in [tr for tr in st if tr.stats.location != select_loc]:
st.remove(tr)
# Data fill for current TIME INTERVAL!
fill = psutils.get_fill(st,
starttime=t0,
endtime=t0 + dt.timedelta(minutes=endminutes))
if fill < minfill:
# not enough data
raise pserrors.CannotPreprocess("{:.0f}% fill".format(fill * 100))
# Merging traces, FILLING GAPS WITH LINEAR INTERP
if len(st) > 1:
st.split()
st.merge(fill_value='interpolate')
# st.merge()
# if such and such about splitting occurs, then inact the split() fn
#st.plot()
#st.split()
trace = st[0]
return trace
def preprocess_trace(self, trace, paz=None, verbose=False):
"""
Preprocesses a trace (so that it is ready to be cross-correlated),
by applying the following steps:
- removal of instrument response, mean and trend
- band-pass filtering between *freqmin*-*freqmax*
- downsampling to *period_resample* secs
- time-normalization (one-bit normalization or normalization
by the running mean in the earthquake frequency band)
- spectral whitening (if running mean normalization)
Raises CannotPreprocess exception if:
- trace only contains 0 (happens sometimes...)
- a normalization weight is 0 or NaN
- a Nan appeared in trace data
Note that the processing steps are performed in-place.
@type trace: L{Trace}
@param paz: poles and zeros of instrumental response
(set None if response is directly attached to trace)
"""
# =======================================
# Check if any data to work with to begin
# =======================================
#print type(trace)
#print "trace: ", trace
if np.all(trace.data == 0.0):
# no data -> skipping trace
raise pserrors.CannotPreprocess("Only zeros")
# ==========================
# Remove instrument response
# ==========================
if RESP_REMOVE:
t1 = dt.datetime.now()
trace = self.remove_resp(trace, paz=paz)
delta = (dt.datetime.now() - t1).total_seconds()
if verbose:
print "\nRemoved response in {:.1f} seconds".format(delta)
#Initialise the AutoTrigger class for the current trace
if HIGHAMP_REMOVE:
# this check finds high amplitude noise creates a list of where it is located in the signal
TRIGGER = AutoTrigger(trace, freqmin=FREQMIN, freqmax=FREQMAX)
UTC_events = TRIGGER.trigger_times(check=True)
if UTC_events is not None:
# set the application to remove these parts from the trace!
trace_new = None
for event in UTC_events:
# event is a list of length 2, start and end times
# use trace.trim(starttime=x, endtime=y)
# trace 1 is from trace start until the event starttime
trace1 = trace.trim(starttime=trace.stats.starttime,
endtime=event[0])
trace2 = trace.trim(starttime=event[1],
endtime=trace.stats.endtime)
if trace_new is None:
trace_new = trace1 + trace2
else:
trace_new += (trace1 + trace2)
# trace_new has removed all high amp. noise from trace original
trace = trace_new
print trace.data
# ==============================================
# Check trace data completeness is above MINFILL
# ==============================================
if COMPLETENESS:
# Data fill for current TIME INTERVAL!
# CHECK THAT THE get_fill_trace function works!
fill = psutils.get_fill_trace(trace)
if fill < MINFILL:
# not enough data
raise pserrors.CannotPreprocess("{:.0f}% fill"\
.format(fill*100))
# ========================
# Trim, demean and detrend
# ========================
if TDD:
t1 = dt.datetime.now()
midt = trace.stats.starttime + (trace.stats.endtime -
trace.stats.starttime) / 2.0
t0 = UTCDateTime(midt.date) # date of trace, at time 00h00m00s
trace.trim(starttime=t0, endtime=t0 + dt.timedelta(days=1))
trace.detrend(type='constant')
trace.detrend(type='linear')
delta = (dt.datetime.now() - t1).total_seconds()
if verbose:
print "Processed trim in {:.1f} seconds".format(delta)
#plt.figure()
#plt.title("TDD")
#plt.plot(trace.data)
#plt.show()
#plt.clf()
# =========
# Band-pass
# =========
if BANDPASS:
t0 = dt.datetime.now()
trace.filter(type="bandpass",
freqmin=self.freqmin,
freqmax=self.freqmax,
corners=self.corners,
zerophase=self.zerophase)
delta = (dt.datetime.now() - t0).total_seconds()
if verbose:
print "Processed filters in {:.1f} seconds".format(delta)
#plt.figure()
#plt.title("Bandpass")
#plt.plot(trace.data)
#plt.show()
#plt.clf()
# ============
# Downsampling
# ============
if DOWNSAMPLE:
if abs(1.0 / trace.stats.sampling_rate -
self.period_resample) > EPS:
psutils.resample(trace, dt_resample=self.period_resample)
#plt.figure()
#plt.title("DOWNSAMPLE")
#plt.plot(trace.data)
#plt.show()
#plt.clf()
# ==================
# Time normalization
# ==================
if TIME_NOMALISATION:
t0 = dt.datetime.now()
# keeping a copy of the trace for weights of time-normalization
trcopy = trace.copy()
trace = self.time_norm(trace, trcopy)
delta = (dt.datetime.now() - t0).total_seconds()
if verbose:
print "Processed time-normalisation in {:.1f} seconds"\
.format(delta)
#plt.figure()
#plt.title("NORMALISATION")
#plt.plot(trace.data)
#plt.show()
#plt.clf()
# ==================
# Spectral whitening
# ==================
if SPEC_WHITENING:
t0 = dt.datetime.now()
trace = self.spectral_whitening(trace)
delta = (dt.datetime.now() - t0).total_seconds()
if verbose:
print "Processed spectral whitening in {:.1f} seconds".\
format(delta)
#plt.figure()
#plt.title("SPECTRAL WHITENING")
#plt.plot(trace.data)
#plt.show()
#plt.clf()
# ==============================================
# Verifying that we don't have nan in trace data
# ==============================================
if np.any(np.isnan(trace.data)):
raise pserrors.CannotPreprocess("Got NaN in trace data")