logging.info(
"\n\
Input filename = %s\n\
Output basename = %s\n\
Start time = %s\n\
End time = %s\n"
% (options.data_file, options.output_file, options.starttime, options.endtime)
)
# read waveform between time limits
wf = Waveform()
wf.read_from_file(options.data_file, starttime=tdeb, endtime=tfin)
dt = wf.delta
x = wf.values
print(wf.stream)
# set up parameters for kurtogram analysis
N = len(x)
N2 = np.log2(N) - 7
nlevel = int(np.fix(N2))
c, flower, fupper = Fast_Kurtogram(x, nlevel, options.verbose, Fs=1 / dt, opt2=1)
logging.info("Frequency band for best kurtogram : %.2f Hz - %.2f Hz" % (flower, fupper))
filt_name = "filt_%s" % options.output_file
kurt_name = "kurt_%s" % options.output_file
wf.bp_filter(flower, fupper, rmean=True, taper=True)
wf.write_to_file_filled(filt_name, format="MSEED")
wf.process_kurtosis(100 * dt, recursive=True, post_taper=True)
wf.write_to_file_filled(kurt_name, format="MSEED")
def do_SDS_processing_setup_and_run(opdict):
"""
Does all the processing for an SDS archive. All options are given within a
*WavlocOptions* object. Steps are:
* reading data
* filtering
* resampling if requested
* applying kurtosis processing
* calculating kurtosis gradient if requested
* convolving with gaussian if requested
* writing processed files at each stage
:param opdict: Dictionary of options. Refers the the opdict attribute of
*WavelocOptions* objects.
:type opdict: dictionary
"""
base_path = opdict['base_path']
data_dir = os.path.join(base_path, 'data', opdict['datadir'])
filter_c1 = opdict['c1']
filter_c2 = opdict['c2']
kurt_window = opdict['kwin']
dataglob = opdict['dataglob']
kurtglob = opdict['kurtglob']
gradglob = opdict['gradglob']
if opdict['gauss']:
gaussglob = opdict['gaussglob']
# start and end time to process
start_time = utcdatetime.UTCDateTime(opdict['starttime'])
end_time = utcdatetime.UTCDateTime(opdict['endtime'])
# if have a channel file then read it
if 'channel_file' in opdict:
fname = os.path.join(base_path, 'lib', opdict['channel_file'])
triplet_list = read_channel_file(fname)
else:
# else make triplet list from net, sta, comp lists
triplet_list = []
net_list = opdict['net_list'].split(',')
sta_list = opdict['sta_list'].split(',')
comp_list = opdict['comp_list'].split(',')
for net in net_list:
for sta in sta_list:
for comp in comp_list:
triplet_list.append((net, sta, comp))
# loop over data
for net, sta, comp in triplet_list:
full_path = os.path.join(data_dir, net, sta, "%s.D" % comp)
logging.debug("Full path : %s" % full_path)
if os.path.exists(full_path):
# construct the base filename for output and create the wf object
filt_filename = os.path.join(
data_dir, "%s.%s.%s.%s.%s" %
(start_time.isoformat(), net, sta, comp, dataglob[1:]))
logging.debug("Processing to create %s" % (filt_filename))
wf = Waveform()
# read and process the data
try:
# read and filter data
wf.read_from_SDS(data_dir,
net,
sta,
comp,
starttime=start_time,
endtime=end_time)
wf.bp_filter(filter_c1, filter_c2, rmean=True, taper=True)
if opdict['resample']:
wf.resample(opdict['fs'])
wf.write_to_file_filled(filt_filename,
format='MSEED',
fill_value=0)
# do kurtosis processing
kurt_filename = os.path.join(
data_dir, "%s.%s.%s.%s.%s" %
(start_time.isoformat(), net, sta, comp, kurtglob[1:]))
logging.debug("Processing to create %s" % (kurt_filename))
wf.process_kurtosis(kurt_window,
recursive=opdict['krec'],
pre_taper=True,
post_taper=True)
wf.write_to_file_filled(kurt_filename,
format='MSEED',
fill_value=0)
# calculate kurtosis gradient if requested
if opdict['kderiv']:
kurt_grad_filename = os.path.join(
data_dir, "%s.%s.%s.%s.%s" %
(start_time.isoformat(), net, sta, comp, gradglob[1:]))
logging.debug("Processing to create %s" %
(kurt_grad_filename))
wf.take_positive_derivative(pre_taper=True,
post_taper=True)
wf.write_to_file_filled(kurt_grad_filename,
format='MSEED',
fill_value=0)
# do convolution with gaussian if requested
if opdict['gauss']:
thres = opdict['gthreshold']
mu = opdict['mu']
sigma = opdict['sigma']
gauss_filename = os.path.join(
data_dir,
"%s.%s.%s.%s.%s" % (start_time.isoformat(), net, sta,
comp, gaussglob[1:]))
logging.debug("Processing to create %s" % (gauss_filename))
wf.process_gaussian(thres, mu, sigma)
wf.write_to_file_filled(gauss_filename,
format='MSEED',
fill_value=0)
except UserWarning:
logging.info('No data within time limits for %s %s %s' %
(net, sta, comp))
def do_SDS_processing_setup_and_run(opdict):
"""
Does all the processing for an SDS archive. All options are given within a
*WavlocOptions* object. Steps are:
* reading data
* filtering
* resampling if requested
* applying kurtosis processing
* calculating kurtosis gradient if requested
* convolving with gaussian if requested
* writing processed files at each stage
:param opdict: Dictionary of options. Refers the the opdict attribute of
*WavelocOptions* objects.
:type opdict: dictionary
"""
base_path = opdict['base_path']
data_dir = os.path.join(base_path, 'data', opdict['datadir'])
filter_c1 = opdict['c1']
filter_c2 = opdict['c2']
kurt_window = opdict['kwin']
dataglob = opdict['dataglob']
kurtglob = opdict['kurtglob']
gradglob = opdict['gradglob']
if opdict['gauss']:
gaussglob = opdict['gaussglob']
# start and end time to process
start_time = utcdatetime.UTCDateTime(opdict['starttime'])
end_time = utcdatetime.UTCDateTime(opdict['endtime'])
# if have a channel file then read it
if 'channel_file' in opdict:
fname = os.path.join(base_path, 'lib', opdict['channel_file'])
triplet_list = read_channel_file(fname)
else:
# else make triplet list from net, sta, comp lists
triplet_list = []
net_list = opdict['net_list'].split(',')
sta_list = opdict['sta_list'].split(',')
comp_list = opdict['comp_list'].split(',')
for net in net_list:
for sta in sta_list:
for comp in comp_list:
triplet_list.append((net, sta, comp))
# loop over data
for net, sta, comp in triplet_list:
full_path = os.path.join(data_dir, net, sta, "%s.D" % comp)
logging.debug("Full path : %s" % full_path)
if os.path.exists(full_path):
# construct the base filename for output and create the wf object
filt_filename = os.path.join(data_dir,
"%s.%s.%s.%s.%s" %
(start_time.isoformat(),
net, sta, comp, dataglob[1:]))
logging.debug("Processing to create %s" % (filt_filename))
wf = Waveform()
# read and process the data
try:
# read and filter data
wf.read_from_SDS(data_dir, net, sta, comp,
starttime=start_time, endtime=end_time)
wf.bp_filter(filter_c1, filter_c2, rmean=True, taper=True)
if opdict['resample']:
wf.resample(opdict['fs'])
wf.write_to_file_filled(filt_filename, format='MSEED',
fill_value=0)
# do kurtosis processing
kurt_filename = os.path.join(data_dir,
"%s.%s.%s.%s.%s" %
(start_time.isoformat(),
net, sta, comp, kurtglob[1:]))
logging.debug("Processing to create %s" % (kurt_filename))
wf.process_kurtosis(kurt_window, recursive=opdict['krec'],
pre_taper=True, post_taper=True)
wf.write_to_file_filled(kurt_filename, format='MSEED',
fill_value=0)
# calculate kurtosis gradient if requested
if opdict['kderiv']:
kurt_grad_filename = os.path.join(data_dir,
"%s.%s.%s.%s.%s" %
(start_time.isoformat(),
net, sta, comp,
gradglob[1:]))
logging.debug("Processing to create %s" %
(kurt_grad_filename))
wf.take_positive_derivative(pre_taper=True,
post_taper=True)
wf.write_to_file_filled(kurt_grad_filename, format='MSEED',
fill_value=0)
# do convolution with gaussian if requested
if opdict['gauss']:
thres = opdict['gthreshold']
mu = opdict['mu']
sigma = opdict['sigma']
gauss_filename = os.path.join(data_dir,
"%s.%s.%s.%s.%s" %
(start_time.isoformat(),
net, sta, comp,
gaussglob[1:]))
logging.debug("Processing to create %s" % (gauss_filename))
wf.process_gaussian(thres, mu, sigma)
wf.write_to_file_filled(gauss_filename, format='MSEED',
fill_value=0)
except UserWarning:
logging.info('No data within time limits for %s %s %s' %
(net, sta, comp))
starttime = utcdatetime.UTCDateTime("2010-10-14T00:14:00.0Z")
endtime = utcdatetime.UTCDateTime("2010-10-14T00:18:00.0Z")
c1 = 4.0
c2 = 10.0
kwin = 3.0
logging.info('Computing full kurtosis kwin=%.2fs' % kwin)
wf_raw = Waveform()
wf_raw.read_from_file(test_file,
starttime=starttime,
endtime=endtime,
rmean=True,
taper=True)
wf_raw.bp_filter(freqmin=c1, freqmax=c2)
wf_kurt = deepcopy(wf_raw)
wf_kurt.process_kurtosis(kwin)
tr1 = np.array(wf_kurt.trace.data)
tr1 = tr1 / np.max(tr1)
tr_len = len(tr1)
misfits = []
k_factors = np.linspace(0.01, 0.2, 30)
print k_factors
for k_factor in k_factors:
k_rec = kwin * k_factor
logging.info('Computing recursive kurtosis kwin=%.2fs' % k_rec)
def kurto(origin_time, info, opdict):
"""
Finds for each Waveloc event and for each station the best filtering
parameters for kurtosis computation.
Writes them into the dictionary info.
:param origin_time: origin time of the signal
:param info: dictionary of parameters
:param opdict: dictionary of the Waveloc parameters and options
:type origin_time: utcdatetime
:type info: dictionary
:type opdict: dictionary
:rtype: dictionary
:returns: info
"""
verbose = opdict['verbose']
kwin = opdict['kwin']
start_time = origin_time-5.0
end_time = origin_time+20.0
dt = info['dt']
# Trace
x = waveval(info['data_ini'], start_time, end_time, dt, info['tdeb_data'])
if not x.any() and x.all():
return info
# Initial kurtosis (trace filtered between 4-10Hz)
kurtx = waveval(info['kurt_ini'], start_time, end_time, dt,
info['tdeb_kurt'])
kurtx = smooth(kurtx)
N = len(x)
N2 = np.log2(N)-7
nlevel = int(np.fix(N2))
snr_ref = np.max(np.abs(x))/np.mean(np.abs(x))
snr_kurt_ref = np.max(np.abs(kurtx))/np.mean(np.abs(kurtx))
kmax_ref = np.max(kurtx) # maximum of the kurtosis
# Compute the kurtogram and keep best frequencies
if verbose:
import matplotlib.gridspec as gridspec
G = gridspec.GridSpec(3, 2)
fig = plt.figure(figsize=(15, 6))
fig.set_facecolor('white')
fig.add_subplot(G[:, 0])
Kwav, Level_w, freq_w, c, f_lower, f_upper = \
Fast_Kurtogram(np.array(x, dtype=float), nlevel, verbose, Fs=1/dt,
opt2=1)
# Comparison of the kurtosis computed in the new frequency band and the old
# one (criterion : snr, kmax)
# 1. Read the initial data
wf = Waveform()
wf.read_from_file(info['data_file'], starttime=start_time-kwin,
endtime=end_time+kwin)
nbpts = int(kwin*1./dt)
# 2. Filter the trace with kurtogram frequencies
wf.bp_filter(f_lower, f_upper)
x_filt = wf.values
x_filt = x_filt[nbpts:-nbpts]
# 3. Compute the kurtosis
wf.process_kurtosis(kwin, recursive=opdict['krec'])
new_kurtx = wf.values
if opdict['krec']:
new_kurtx = new_kurtx[nbpts+1:-nbpts-1]
else:
new_kurtx = new_kurtx[:-nbpts-1]
snr = np.max(np.abs(x_filt))/np.mean(np.abs(x_filt))
snr_kurt = np.max(np.abs(new_kurtx))/np.mean(np.abs(new_kurtx))
kmax = np.max(new_kurtx)
if snr > snr_ref and kmax >= kmax_ref:
info['filter'].append((round(f_lower*100)/100, round(f_upper*100)/100))
if 'new_kurt_file' in info:
info = write_file(info, start_time, end_time, new_kurtx)
else:
info['filter'].append((0, 50))
if verbose and snr > 3:
print "snr:", snr, " ; snr_ref:", snr_ref
print "snr new kurtosis:", snr_kurt, " ; snr kurtosis reference:",\
snr_kurt_ref
print "kurtosis max, kurt_ref :", kmax, kmax_ref
plot_trace(fig, G, x, x_filt, kurtx, new_kurtx, info, f_lower,
f_upper, snr, snr_ref, snr_kurt, kmax, kmax_ref,
origin_time)
plt.show()
return info
logging.basicConfig(level=logging.INFO, format='%(levelname)s : %(asctime)s : %(message)s')
base_path=os.getenv('WAVELOC_PATH')
test_file=os.path.join(base_path,'test_data','raw_data','YA.UV15.00.HHZ.MSEED')
starttime=utcdatetime.UTCDateTime("2010-10-14T00:14:00.0Z")
endtime=utcdatetime.UTCDateTime("2010-10-14T00:18:00.0Z")
c1=4.0
c2=10.0
kwin=3.0
logging.info('Computing full kurtosis kwin=%.2fs'%kwin)
wf_raw=Waveform()
wf_raw.read_from_file(test_file,starttime=starttime,endtime=endtime,rmean=True,taper=True)
wf_raw.bp_filter(freqmin=c1, freqmax=c2)
wf_kurt=deepcopy(wf_raw)
wf_kurt.process_kurtosis(kwin)
tr1=np.array(wf_kurt.trace.data)
tr1 = tr1/np.max(tr1)
tr_len=len(tr1)
misfits=[]
k_factors=np.linspace(0.01,0.2,30)
print k_factors
for k_factor in k_factors:
k_rec=kwin*k_factor
logging.info('Computing recursive kurtosis kwin=%.2fs'%k_rec)
def kurto(origin_time, info, opdict):
"""
Finds for each Waveloc event and for each station the best filtering
parameters for kurtosis computation.
Writes them into the dictionary info.
:param origin_time: origin time of the signal
:param info: dictionary of parameters
:param opdict: dictionary of the Waveloc parameters and options
:type origin_time: utcdatetime
:type info: dictionary
:type opdict: dictionary
:rtype: dictionary
:returns: info
"""
verbose = opdict['verbose']
kwin = opdict['kwin']
start_time = origin_time - 5.0
end_time = origin_time + 20.0
dt = info['dt']
# Trace
x = waveval(info['data_ini'], start_time, end_time, dt, info['tdeb_data'])
if not x.any() and x.all():
return info
# Initial kurtosis (trace filtered between 4-10Hz)
kurtx = waveval(info['kurt_ini'], start_time, end_time, dt,
info['tdeb_kurt'])
kurtx = smooth(kurtx)
N = len(x)
N2 = np.log2(N) - 7
nlevel = int(np.fix(N2))
snr_ref = np.max(np.abs(x)) / np.mean(np.abs(x))
snr_kurt_ref = np.max(np.abs(kurtx)) / np.mean(np.abs(kurtx))
kmax_ref = np.max(kurtx) # maximum of the kurtosis
# Compute the kurtogram and keep best frequencies
if verbose:
import matplotlib.gridspec as gridspec
G = gridspec.GridSpec(3, 2)
fig = plt.figure(figsize=(15, 6))
fig.set_facecolor('white')
fig.add_subplot(G[:, 0])
Kwav, Level_w, freq_w, c, f_lower, f_upper = \
Fast_Kurtogram(np.array(x, dtype=float), nlevel, verbose, Fs=1/dt,
opt2=1)
# Comparison of the kurtosis computed in the new frequency band and the old
# one (criterion : snr, kmax)
# 1. Read the initial data
wf = Waveform()
wf.read_from_file(info['data_file'],
starttime=start_time - kwin,
endtime=end_time + kwin)
nbpts = int(kwin * 1. / dt)
# 2. Filter the trace with kurtogram frequencies
wf.bp_filter(f_lower, f_upper)
x_filt = wf.values
x_filt = x_filt[nbpts:-nbpts]
# 3. Compute the kurtosis
wf.process_kurtosis(kwin, recursive=opdict['krec'])
new_kurtx = wf.values
if opdict['krec']:
new_kurtx = new_kurtx[nbpts + 1:-nbpts - 1]
else:
new_kurtx = new_kurtx[:-nbpts - 1]
snr = np.max(np.abs(x_filt)) / np.mean(np.abs(x_filt))
snr_kurt = np.max(np.abs(new_kurtx)) / np.mean(np.abs(new_kurtx))
kmax = np.max(new_kurtx)
if snr > snr_ref and kmax >= kmax_ref:
info['filter'].append(
(round(f_lower * 100) / 100, round(f_upper * 100) / 100))
if 'new_kurt_file' in info:
info = write_file(info, start_time, end_time, new_kurtx)
else:
info['filter'].append((0, 50))
if verbose and snr > 3:
print "snr:", snr, " ; snr_ref:", snr_ref
print "snr new kurtosis:", snr_kurt, " ; snr kurtosis reference:",\
snr_kurt_ref
print "kurtosis max, kurt_ref :", kmax, kmax_ref
plot_trace(fig, G, x, x_filt, kurtx, new_kurtx, info, f_lower, f_upper,
snr, snr_ref, snr_kurt, kmax, kmax_ref, origin_time)
plt.show()
return info
Start time = %s\n\
End time = %s\n' % (options.data_file, options.output_file,
options.starttime, options.endtime))
# read waveform between time limits
wf = Waveform()
wf.read_from_file(options.data_file, starttime=tdeb, endtime=tfin)
dt = wf.delta
x = wf.values
print(wf.stream)
# set up parameters for kurtogram analysis
N = len(x)
N2 = np.log2(N) - 7
nlevel = int(np.fix(N2))
c, flower, fupper = Fast_Kurtogram(x,
nlevel,
options.verbose,
Fs=1 / dt,
opt2=1)
logging.info("Frequency band for best kurtogram : %.2f Hz - %.2f Hz" %
(flower, fupper))
filt_name = "filt_%s" % options.output_file
kurt_name = "kurt_%s" % options.output_file
wf.bp_filter(flower, fupper, rmean=True, taper=True)
wf.write_to_file_filled(filt_name, format='MSEED')
wf.process_kurtosis(100 * dt, recursive=True, post_taper=True)
wf.write_to_file_filled(kurt_name, format='MSEED')
