logging.info( "\n\ Input filename = %s\n\ Output basename = %s\n\ Start time = %s\n\ End time = %s\n" % (options.data_file, options.output_file, options.starttime, options.endtime) ) # read waveform between time limits wf = Waveform() wf.read_from_file(options.data_file, starttime=tdeb, endtime=tfin) dt = wf.delta x = wf.values print(wf.stream) # set up parameters for kurtogram analysis N = len(x) N2 = np.log2(N) - 7 nlevel = int(np.fix(N2)) c, flower, fupper = Fast_Kurtogram(x, nlevel, options.verbose, Fs=1 / dt, opt2=1) logging.info("Frequency band for best kurtogram : %.2f Hz - %.2f Hz" % (flower, fupper)) filt_name = "filt_%s" % options.output_file kurt_name = "kurt_%s" % options.output_file wf.bp_filter(flower, fupper, rmean=True, taper=True) wf.write_to_file_filled(filt_name, format="MSEED") wf.process_kurtosis(100 * dt, recursive=True, post_taper=True) wf.write_to_file_filled(kurt_name, format="MSEED")
def do_SDS_processing_setup_and_run(opdict): """ Does all the processing for an SDS archive. All options are given within a *WavlocOptions* object. Steps are: * reading data * filtering * resampling if requested * applying kurtosis processing * calculating kurtosis gradient if requested * convolving with gaussian if requested * writing processed files at each stage :param opdict: Dictionary of options. Refers the the opdict attribute of *WavelocOptions* objects. :type opdict: dictionary """ base_path = opdict['base_path'] data_dir = os.path.join(base_path, 'data', opdict['datadir']) filter_c1 = opdict['c1'] filter_c2 = opdict['c2'] kurt_window = opdict['kwin'] dataglob = opdict['dataglob'] kurtglob = opdict['kurtglob'] gradglob = opdict['gradglob'] if opdict['gauss']: gaussglob = opdict['gaussglob'] # start and end time to process start_time = utcdatetime.UTCDateTime(opdict['starttime']) end_time = utcdatetime.UTCDateTime(opdict['endtime']) # if have a channel file then read it if 'channel_file' in opdict: fname = os.path.join(base_path, 'lib', opdict['channel_file']) triplet_list = read_channel_file(fname) else: # else make triplet list from net, sta, comp lists triplet_list = [] net_list = opdict['net_list'].split(',') sta_list = opdict['sta_list'].split(',') comp_list = opdict['comp_list'].split(',') for net in net_list: for sta in sta_list: for comp in comp_list: triplet_list.append((net, sta, comp)) # loop over data for net, sta, comp in triplet_list: full_path = os.path.join(data_dir, net, sta, "%s.D" % comp) logging.debug("Full path : %s" % full_path) if os.path.exists(full_path): # construct the base filename for output and create the wf object filt_filename = os.path.join( data_dir, "%s.%s.%s.%s.%s" % (start_time.isoformat(), net, sta, comp, dataglob[1:])) logging.debug("Processing to create %s" % (filt_filename)) wf = Waveform() # read and process the data try: # read and filter data wf.read_from_SDS(data_dir, net, sta, comp, starttime=start_time, endtime=end_time) wf.bp_filter(filter_c1, filter_c2, rmean=True, taper=True) if opdict['resample']: wf.resample(opdict['fs']) wf.write_to_file_filled(filt_filename, format='MSEED', fill_value=0) # do kurtosis processing kurt_filename = os.path.join( data_dir, "%s.%s.%s.%s.%s" % (start_time.isoformat(), net, sta, comp, kurtglob[1:])) logging.debug("Processing to create %s" % (kurt_filename)) wf.process_kurtosis(kurt_window, recursive=opdict['krec'], pre_taper=True, post_taper=True) wf.write_to_file_filled(kurt_filename, format='MSEED', fill_value=0) # calculate kurtosis gradient if requested if opdict['kderiv']: kurt_grad_filename = os.path.join( data_dir, "%s.%s.%s.%s.%s" % (start_time.isoformat(), net, sta, comp, gradglob[1:])) logging.debug("Processing to create %s" % (kurt_grad_filename)) wf.take_positive_derivative(pre_taper=True, post_taper=True) wf.write_to_file_filled(kurt_grad_filename, format='MSEED', fill_value=0) # do convolution with gaussian if requested if opdict['gauss']: thres = opdict['gthreshold'] mu = opdict['mu'] sigma = opdict['sigma'] gauss_filename = os.path.join( data_dir, "%s.%s.%s.%s.%s" % (start_time.isoformat(), net, sta, comp, gaussglob[1:])) logging.debug("Processing to create %s" % (gauss_filename)) wf.process_gaussian(thres, mu, sigma) wf.write_to_file_filled(gauss_filename, format='MSEED', fill_value=0) except UserWarning: logging.info('No data within time limits for %s %s %s' % (net, sta, comp))
def do_SDS_processing_setup_and_run(opdict): """ Does all the processing for an SDS archive. All options are given within a *WavlocOptions* object. Steps are: * reading data * filtering * resampling if requested * applying kurtosis processing * calculating kurtosis gradient if requested * convolving with gaussian if requested * writing processed files at each stage :param opdict: Dictionary of options. Refers the the opdict attribute of *WavelocOptions* objects. :type opdict: dictionary """ base_path = opdict['base_path'] data_dir = os.path.join(base_path, 'data', opdict['datadir']) filter_c1 = opdict['c1'] filter_c2 = opdict['c2'] kurt_window = opdict['kwin'] dataglob = opdict['dataglob'] kurtglob = opdict['kurtglob'] gradglob = opdict['gradglob'] if opdict['gauss']: gaussglob = opdict['gaussglob'] # start and end time to process start_time = utcdatetime.UTCDateTime(opdict['starttime']) end_time = utcdatetime.UTCDateTime(opdict['endtime']) # if have a channel file then read it if 'channel_file' in opdict: fname = os.path.join(base_path, 'lib', opdict['channel_file']) triplet_list = read_channel_file(fname) else: # else make triplet list from net, sta, comp lists triplet_list = [] net_list = opdict['net_list'].split(',') sta_list = opdict['sta_list'].split(',') comp_list = opdict['comp_list'].split(',') for net in net_list: for sta in sta_list: for comp in comp_list: triplet_list.append((net, sta, comp)) # loop over data for net, sta, comp in triplet_list: full_path = os.path.join(data_dir, net, sta, "%s.D" % comp) logging.debug("Full path : %s" % full_path) if os.path.exists(full_path): # construct the base filename for output and create the wf object filt_filename = os.path.join(data_dir, "%s.%s.%s.%s.%s" % (start_time.isoformat(), net, sta, comp, dataglob[1:])) logging.debug("Processing to create %s" % (filt_filename)) wf = Waveform() # read and process the data try: # read and filter data wf.read_from_SDS(data_dir, net, sta, comp, starttime=start_time, endtime=end_time) wf.bp_filter(filter_c1, filter_c2, rmean=True, taper=True) if opdict['resample']: wf.resample(opdict['fs']) wf.write_to_file_filled(filt_filename, format='MSEED', fill_value=0) # do kurtosis processing kurt_filename = os.path.join(data_dir, "%s.%s.%s.%s.%s" % (start_time.isoformat(), net, sta, comp, kurtglob[1:])) logging.debug("Processing to create %s" % (kurt_filename)) wf.process_kurtosis(kurt_window, recursive=opdict['krec'], pre_taper=True, post_taper=True) wf.write_to_file_filled(kurt_filename, format='MSEED', fill_value=0) # calculate kurtosis gradient if requested if opdict['kderiv']: kurt_grad_filename = os.path.join(data_dir, "%s.%s.%s.%s.%s" % (start_time.isoformat(), net, sta, comp, gradglob[1:])) logging.debug("Processing to create %s" % (kurt_grad_filename)) wf.take_positive_derivative(pre_taper=True, post_taper=True) wf.write_to_file_filled(kurt_grad_filename, format='MSEED', fill_value=0) # do convolution with gaussian if requested if opdict['gauss']: thres = opdict['gthreshold'] mu = opdict['mu'] sigma = opdict['sigma'] gauss_filename = os.path.join(data_dir, "%s.%s.%s.%s.%s" % (start_time.isoformat(), net, sta, comp, gaussglob[1:])) logging.debug("Processing to create %s" % (gauss_filename)) wf.process_gaussian(thres, mu, sigma) wf.write_to_file_filled(gauss_filename, format='MSEED', fill_value=0) except UserWarning: logging.info('No data within time limits for %s %s %s' % (net, sta, comp))
tr1 = np.array(wf_kurt.trace.data) tr1 = tr1 / np.max(tr1) tr_len = len(tr1) misfits = [] k_factors = np.linspace(0.01, 0.2, 30) print k_factors for k_factor in k_factors: k_rec = kwin * k_factor logging.info('Computing recursive kurtosis kwin=%.2fs' % k_rec) wf_krec = Waveform() wf_krec.read_from_file(test_file, starttime=starttime, endtime=endtime, rmean=True, taper=True) wf_krec.bp_filter(freqmin=c1, freqmax=c2) wf_krec.process_kurtosis(k_rec, recursive=True) tr2 = np.array(wf_krec.trace.data) tr2 = tr2 / np.max(tr2) logging.info('Computing misfit...') misfit = np.dot((tr1 - tr2), (tr1 - tr2)) logging.info('misfit = %.3f' % misfit) misfits.append((k_factor, misfit)) print misfits
def kurto(origin_time, info, opdict): """ Finds for each Waveloc event and for each station the best filtering parameters for kurtosis computation. Writes them into the dictionary info. :param origin_time: origin time of the signal :param info: dictionary of parameters :param opdict: dictionary of the Waveloc parameters and options :type origin_time: utcdatetime :type info: dictionary :type opdict: dictionary :rtype: dictionary :returns: info """ verbose = opdict['verbose'] kwin = opdict['kwin'] start_time = origin_time-5.0 end_time = origin_time+20.0 dt = info['dt'] # Trace x = waveval(info['data_ini'], start_time, end_time, dt, info['tdeb_data']) if not x.any() and x.all(): return info # Initial kurtosis (trace filtered between 4-10Hz) kurtx = waveval(info['kurt_ini'], start_time, end_time, dt, info['tdeb_kurt']) kurtx = smooth(kurtx) N = len(x) N2 = np.log2(N)-7 nlevel = int(np.fix(N2)) snr_ref = np.max(np.abs(x))/np.mean(np.abs(x)) snr_kurt_ref = np.max(np.abs(kurtx))/np.mean(np.abs(kurtx)) kmax_ref = np.max(kurtx) # maximum of the kurtosis # Compute the kurtogram and keep best frequencies if verbose: import matplotlib.gridspec as gridspec G = gridspec.GridSpec(3, 2) fig = plt.figure(figsize=(15, 6)) fig.set_facecolor('white') fig.add_subplot(G[:, 0]) Kwav, Level_w, freq_w, c, f_lower, f_upper = \ Fast_Kurtogram(np.array(x, dtype=float), nlevel, verbose, Fs=1/dt, opt2=1) # Comparison of the kurtosis computed in the new frequency band and the old # one (criterion : snr, kmax) # 1. Read the initial data wf = Waveform() wf.read_from_file(info['data_file'], starttime=start_time-kwin, endtime=end_time+kwin) nbpts = int(kwin*1./dt) # 2. Filter the trace with kurtogram frequencies wf.bp_filter(f_lower, f_upper) x_filt = wf.values x_filt = x_filt[nbpts:-nbpts] # 3. Compute the kurtosis wf.process_kurtosis(kwin, recursive=opdict['krec']) new_kurtx = wf.values if opdict['krec']: new_kurtx = new_kurtx[nbpts+1:-nbpts-1] else: new_kurtx = new_kurtx[:-nbpts-1] snr = np.max(np.abs(x_filt))/np.mean(np.abs(x_filt)) snr_kurt = np.max(np.abs(new_kurtx))/np.mean(np.abs(new_kurtx)) kmax = np.max(new_kurtx) if snr > snr_ref and kmax >= kmax_ref: info['filter'].append((round(f_lower*100)/100, round(f_upper*100)/100)) if 'new_kurt_file' in info: info = write_file(info, start_time, end_time, new_kurtx) else: info['filter'].append((0, 50)) if verbose and snr > 3: print "snr:", snr, " ; snr_ref:", snr_ref print "snr new kurtosis:", snr_kurt, " ; snr kurtosis reference:",\ snr_kurt_ref print "kurtosis max, kurt_ref :", kmax, kmax_ref plot_trace(fig, G, x, x_filt, kurtx, new_kurtx, info, f_lower, f_upper, snr, snr_ref, snr_kurt, kmax, kmax_ref, origin_time) plt.show() return info
wf_raw.bp_filter(freqmin=c1, freqmax=c2) wf_kurt=deepcopy(wf_raw) wf_kurt.process_kurtosis(kwin) tr1=np.array(wf_kurt.trace.data) tr1 = tr1/np.max(tr1) tr_len=len(tr1) misfits=[] k_factors=np.linspace(0.01,0.2,30) print k_factors for k_factor in k_factors: k_rec=kwin*k_factor logging.info('Computing recursive kurtosis kwin=%.2fs'%k_rec) wf_krec=Waveform() wf_krec.read_from_file(test_file,starttime=starttime,endtime=endtime,rmean=True,taper=True) wf_krec.bp_filter(freqmin=c1, freqmax=c2) wf_krec.process_kurtosis(k_rec,recursive=True) tr2=np.array(wf_krec.trace.data) tr2 = tr2/np.max(tr2) logging.info('Computing misfit...') misfit=np.dot((tr1-tr2),(tr1-tr2)) logging.info('misfit = %.3f'%misfit) misfits.append((k_factor,misfit)) print misfits
def kurto(origin_time, info, opdict): """ Finds for each Waveloc event and for each station the best filtering parameters for kurtosis computation. Writes them into the dictionary info. :param origin_time: origin time of the signal :param info: dictionary of parameters :param opdict: dictionary of the Waveloc parameters and options :type origin_time: utcdatetime :type info: dictionary :type opdict: dictionary :rtype: dictionary :returns: info """ verbose = opdict['verbose'] kwin = opdict['kwin'] start_time = origin_time - 5.0 end_time = origin_time + 20.0 dt = info['dt'] # Trace x = waveval(info['data_ini'], start_time, end_time, dt, info['tdeb_data']) if not x.any() and x.all(): return info # Initial kurtosis (trace filtered between 4-10Hz) kurtx = waveval(info['kurt_ini'], start_time, end_time, dt, info['tdeb_kurt']) kurtx = smooth(kurtx) N = len(x) N2 = np.log2(N) - 7 nlevel = int(np.fix(N2)) snr_ref = np.max(np.abs(x)) / np.mean(np.abs(x)) snr_kurt_ref = np.max(np.abs(kurtx)) / np.mean(np.abs(kurtx)) kmax_ref = np.max(kurtx) # maximum of the kurtosis # Compute the kurtogram and keep best frequencies if verbose: import matplotlib.gridspec as gridspec G = gridspec.GridSpec(3, 2) fig = plt.figure(figsize=(15, 6)) fig.set_facecolor('white') fig.add_subplot(G[:, 0]) Kwav, Level_w, freq_w, c, f_lower, f_upper = \ Fast_Kurtogram(np.array(x, dtype=float), nlevel, verbose, Fs=1/dt, opt2=1) # Comparison of the kurtosis computed in the new frequency band and the old # one (criterion : snr, kmax) # 1. Read the initial data wf = Waveform() wf.read_from_file(info['data_file'], starttime=start_time - kwin, endtime=end_time + kwin) nbpts = int(kwin * 1. / dt) # 2. Filter the trace with kurtogram frequencies wf.bp_filter(f_lower, f_upper) x_filt = wf.values x_filt = x_filt[nbpts:-nbpts] # 3. Compute the kurtosis wf.process_kurtosis(kwin, recursive=opdict['krec']) new_kurtx = wf.values if opdict['krec']: new_kurtx = new_kurtx[nbpts + 1:-nbpts - 1] else: new_kurtx = new_kurtx[:-nbpts - 1] snr = np.max(np.abs(x_filt)) / np.mean(np.abs(x_filt)) snr_kurt = np.max(np.abs(new_kurtx)) / np.mean(np.abs(new_kurtx)) kmax = np.max(new_kurtx) if snr > snr_ref and kmax >= kmax_ref: info['filter'].append( (round(f_lower * 100) / 100, round(f_upper * 100) / 100)) if 'new_kurt_file' in info: info = write_file(info, start_time, end_time, new_kurtx) else: info['filter'].append((0, 50)) if verbose and snr > 3: print "snr:", snr, " ; snr_ref:", snr_ref print "snr new kurtosis:", snr_kurt, " ; snr kurtosis reference:",\ snr_kurt_ref print "kurtosis max, kurt_ref :", kmax, kmax_ref plot_trace(fig, G, x, x_filt, kurtx, new_kurtx, info, f_lower, f_upper, snr, snr_ref, snr_kurt, kmax, kmax_ref, origin_time) plt.show() return info
Start time = %s\n\ End time = %s\n' % (options.data_file, options.output_file, options.starttime, options.endtime)) # read waveform between time limits wf = Waveform() wf.read_from_file(options.data_file, starttime=tdeb, endtime=tfin) dt = wf.delta x = wf.values print(wf.stream) # set up parameters for kurtogram analysis N = len(x) N2 = np.log2(N) - 7 nlevel = int(np.fix(N2)) c, flower, fupper = Fast_Kurtogram(x, nlevel, options.verbose, Fs=1 / dt, opt2=1) logging.info("Frequency band for best kurtogram : %.2f Hz - %.2f Hz" % (flower, fupper)) filt_name = "filt_%s" % options.output_file kurt_name = "kurt_%s" % options.output_file wf.bp_filter(flower, fupper, rmean=True, taper=True) wf.write_to_file_filled(filt_name, format='MSEED') wf.process_kurtosis(100 * dt, recursive=True, post_taper=True) wf.write_to_file_filled(kurt_name, format='MSEED')