def read_multiplet(filename,
idx,
return_tp=False,
db_path=cfg.dbpath,
db_path_T='template_db_1/',
db_path_M='matched_filter_1/'):
"""
read_multiplet(filename, idx, db_path=cfg.dbpath) \n
"""
S = Stream()
f_meta = db_path + db_path_M + filename + 'meta.h5'
fm = h5.File(f_meta, 'r')
T = read_template('template{:d}'.format(fm['template_id'][0]),
db_path=db_path + db_path_T)
f_wave = db_path + db_path_M + filename + 'wav.h5'
fw = h5.File(f_wave, 'r')
waveforms = fw['waveforms'][idx, :, :, :]
fw.close()
#---------------------------------
stations = fm['stations'][:].astype('U')
components = fm['components'][:].astype('U')
ns = len(stations)
nc = len(components)
#---------------------------------
date = udt(fm['origin_times'][idx])
for s in range(ns):
for c in range(nc):
S += Trace(data=waveforms[s, c, :])
S[-1].stats['station'] = stations[s]
S[-1].stats['channel'] = components[c]
S[-1].stats['sampling_rate'] = cfg.sampling_rate
S[-1].stats.starttime = date
S.s_moveouts = T.metadata['s_moveouts']
S.p_moveouts = T.metadata['p_moveouts']
S.source_idx = T.metadata['source_idx']
S.template_ID = T.metadata['template_idx']
S.latitude = T.metadata['latitude']
S.longitude = T.metadata['longitude']
S.depth = T.metadata['depth']
S.corr = fm['correlation_coefficients'][idx]
S.stations = stations.tolist()
S.components = components.tolist()
fm.close()
if return_tp:
return S, T
else:
return S
def SVDWF_multiplets_bulk(template_id, db_path=autodet.cfg.dbpath, db_path_M='matched_filter_1/', db_path_T='template_db_1/', \
WAVEFORMS=None, best=False, normRMS=True, \
n_singular_values=5, max_freq=autodet.cfg.max_freq, attach_raw_data=False):
from obspy import Stream, Trace
from scipy.linalg import svd
from scipy.signal import wiener
#-----------------------------------------------------------------------------------------------
T = autodet.db_h5py.read_template('template{:d}'.format(template_id), db_path=db_path+db_path_T)
#-----------------------------------------------------------------------------------------------
files_all = glob.glob(db_path + db_path_M + '*multiplets_*meta.h5')
files = []
#------------------------------
S = Stream()
CC = []
tid_str = str(template_id)
t1 = give_time()
for file in files_all:
with h5.File(file, mode='r') as f:
if tid_str in f.keys():
files.append(file[:-len('meta.h5')])
CC.extend(f[tid_str]['correlation_coefficients'][()].tolist())
CC = np.float32(CC)
t2 = give_time()
print('{:.2f} s to retrieve the correlation coefficients.'.format(t2-t1))
if len(files) == 0:
print("None multiplet for template {:d} !! Return None".format(template_id))
return None
with h5.File(files[0] + 'meta.h5', mode='r') as f:
S.stations = f[tid_str]['stations'][()].astype('U').tolist()
S.components = f[tid_str]['components'][()].astype('U').tolist()
ns = len(S.stations)
nc = len(S.components)
S.latitude = T.metadata['latitude']
S.longitude = T.metadata['longitude']
S.depth = T.metadata['depth']
#------------------------------
#----------------------------------------------
if WAVEFORMS is None:
CC = np.sort(CC)
if len(CC) > 300:
CC_thres = CC[-101]
elif len(CC) > 70:
CC_thres = CC[int(7./10.*len(CC))] # the best 30%
elif len(CC) > 30:
CC_thres = np.median(CC) # the best 50%
elif len(CC) > 10:
CC_thres = np.percentile(CC, 33.) # the best 66% detections
else:
CC_thres = 0.
Nstack = np.zeros((ns, nc), dtype=np.float32)
WAVEFORMS = []
Nmulti = 0
t1 = give_time()
for file in files:
if best:
with h5.File(file + 'meta.h5', mode='r') as fm:
selection = np.where(fm[tid_str]['correlation_coefficients'][:] > CC_thres)[0]
if selection.size == 0:
continue
with h5.File(file + 'wav.h5', mode='r') as fw:
WAVEFORMS.append(fw[tid_str]['waveforms'][selection, :, :, :])
else:
with h5.File(file + 'wav.h5', mode='r') as fw:
WAVEFORMS.append(fw[tid_str]['waveforms'][()])
Nmulti += WAVEFORMS[-1].shape[0]
for m in range(WAVEFORMS[-1].shape[0]):
for s in range(ns):
for c in range(nc):
if normRMS:
norm = np.sqrt(np.var(WAVEFORMS[-1][m,s,c,:]))
else:
norm =1.
if norm != 0.:
WAVEFORMS[-1][m,s,c,:] /= norm
t2 = give_time()
print('{:.2f} s to retrieve the waveforms.'.format(t2-t1))
elif normRMS:
for m in range(WAVEFORMS.shape[0]):
for s in range(ns):
for c in range(nc):
norm = np.sqrt(np.var(WAVEFORMS[m,s,c,:]))
if norm != 0.:
WAVEFORMS[m,s,c,:] /= norm
else:
pass
WAVEFORMS = np.vstack(WAVEFORMS)
WAVEFORMS = WAVEFORMS.reshape(-1, ns, nc, WAVEFORMS.shape[-1])
print(WAVEFORMS.shape)
filtered_data = np.zeros_like(WAVEFORMS)
for s in range(ns):
for c in range(nc):
filtered_data[:,s,c,:] = SVDWF(WAVEFORMS[:,s,c,:], n_singular_values, max_freq=max_freq)
#filtered_data[:,s,c,:] = spectral_filtering(WAVEFORMS[:,s,c,:], SNR_thres=5., max_freq=max_freq)
mean = np.mean(filtered_data[:,s,c,:], axis=0)
mean /= np.abs(mean).max()
S += Trace(data=mean)
S[-1].stats.station = S.stations[s]
S[-1].stats.channel = S.components[c]
S[-1].stats.sampling_rate = autodet.cfg.sampling_rate
S.data = filtered_data
if attach_raw_data:
S.raw_data = WAVEFORMS
S.Nmulti = Nmulti
return S
def spectral_filtering_detections(tid, db_path_T='template_db_1/', db_path_M='matched_filter_1/', db_path=autodet.cfg.dbpath, SNR_thres=5., WAVEFORMS=None, normRMS=True, best=True):
from subprocess import Popen, PIPE
from obspy import Stream, Trace
#-----------------------------------------------------------------------------------------------
T = autodet.db_h5py.read_template('template{:d}'.format(tid), db_path=db_path+db_path_T)
#-----------------------------------------------------------------------------------------------
print("Looking for {}{:d}_*".format(db_path + db_path_M + '*multiplets', tid))
files_list = Popen('ls '+db_path+db_path_M+'*multiplets{:d}_*'.format(tid), stdout=PIPE, shell=True).stdout
line = files_list.readline()[:-1]
files = []
while len(line) != 0:
files.append(line.decode('utf-8'))
line = files_list.readline()[:-1]
i = 0
Nsamp = 0
ns = 0
S = Stream()
#------------- retrieve metadata ---------------
while True:
try:
wav = files[i][-len('wav.h5'):] == 'wav.h5'
if wav:
if Nsamp == 0:
with h5.File(files[i], mode='r') as fwav0:
Nsamp = fwav0['waveforms'][:,:,:,:].shape[-1]
i += 1
else:
with h5.File(files[i], mode='r') as fm0:
if len(fm0['origin_times']) == 0:
i += 1
continue
else:
i += 1
nc = len(fm0['components'][()])
ns = len(fm0['stations'][()])
S.stations = fm0['stations'][()].astype('U').tolist()
S.components = fm0['components'][()].astype('U').tolist()
S.latitude = T.metadata['latitude']
S.longitude = T.metadata['longitude']
S.depth = T.metadata['depth']
S.template_ID = tid
if ns != 0 and Nsamp != 0:
break
except IndexError:
print("None multiplet for template {:d} !! Return None".format(tid))
return None
#----------------------------------------------
if WAVEFORMS is None:
CC = np.zeros(0, dtype=np.float32)
if best:
for file in files:
if file[-len('meta.h5'):] != 'meta.h5':
continue
with h5.File(file, mode='r') as fm:
if len(fm['correlation_coefficients']) == 0:
continue
else:
CC = np.hstack((CC, fm['correlation_coefficients'][:]))
CC = np.sort(CC)
#CC_thres = np.sort(CC)[-min(5, len(CC))]
if len(CC) > 300:
CC_thres = CC[-101]
elif len(CC) > 70:
CC_thres = CC[int(7./10.*len(CC))] # the best 30%
elif len(CC) > 30:
CC_thres = np.median(CC) # the best 50%
elif len(CC) > 10:
CC_thres = np.percentile(CC, 33.) # the best 66% detections
else:
CC_thres = 0.
Nstack = np.zeros((ns, nc), dtype=np.float32)
WAVEFORMS = np.zeros((0,ns,nc,Nsamp), dtype=np.float32)
Nmulti = 0
for file in files:
if file[-len('wav.h5'):] != 'wav.h5':
continue
with h5.File(file, mode='r') as fw:
if len(fw['waveforms']) == 0:
continue
else:
if best:
with h5.File(file[:-len('wav.h5')]+'meta.h5', mode='r') as fm:
selection = np.where(fm['correlation_coefficients'][:] > CC_thres)[0]
if selection.size == 0:
continue
waves = np.zeros((selection.size, ns, nc, Nsamp), dtype=np.float32)
waves[:,:,:,:] = fw['waveforms'][selection,:,:,:]
else:
waves = fw['waveforms'][:,:,:,:]
Nmulti += waves.shape[0]
for m in range(waves.shape[0]):
for s in range(ns):
for c in range(nc):
if normRMS:
norm = np.sqrt(np.var(waves[m,s,c,:]))
else:
norm =1.
if norm != 0.:
waves[m,s,c,:] /= norm
WAVEFORMS = np.vstack((WAVEFORMS, waves))
elif normRMS:
for m in range(WAVEFORMS.shape[0]):
for s in range(ns):
for c in range(nc):
norm = np.sqrt(np.var(WAVEFORMS[m,s,c,:]))
if norm != 0.:
WAVEFORMS[m,s,c,:] /= norm
else:
pass
filtered_waveforms = np.zeros((ns, nc, Nsamp), dtype=np.float32)
for s in range(ns):
for c in range(nc):
filtered_waveforms[s, c, :] = np.sum(spectral_filtering(WAVEFORMS[:, s, c, :], SNR_thres=SNR_thres), axis=0)
return filtered_waveforms
def SVDWF_multiplets_test(template_id, db_path=autodet.cfg.dbpath, db_path_M='matched_filter_2/', db_path_T='template_db_2/', WAVEFORMS=None, normRMS=True, Nsing_values=5, max_freq=autodet.cfg.max_freq, attach_raw_data=False):
from subprocess import Popen, PIPE
from obspy import Stream, Trace
from scipy.linalg import svd
from scipy.signal import wiener
#-----------------------------------------------------------------------------------------------
T = autodet.db_h5py.read_template('template{:d}'.format(template_id), db_path=db_path+db_path_T)
#-----------------------------------------------------------------------------------------------
print("Looking for {}{:d}_*".format(db_path + db_path_M + '*multiplets', template_id))
files_list = Popen('ls '+db_path+db_path_M+'*multiplets{:d}_*'.format(template_id), stdout=PIPE, shell=True).stdout
line = files_list.readline()[:-1]
files = []
while len(line) != 0:
files.append(line.decode('utf-8'))
line = files_list.readline()[:-1]
i = 0
Nsamp = 0
ns = 0
S = Stream()
#------------- retrieve metadata ---------------
while True:
try:
wav = files[i][-len('wav.h5'):] == 'wav.h5'
if wav:
if Nsamp == 0:
with h5.File(files[i], mode='r') as fwav0:
Nsamp = fwav0['waveforms'][:,:,:,:].shape[-1]
i += 1
else:
with h5.File(files[i], mode='r') as fm0:
if len(fm0['origin_times']) == 0:
i += 1
continue
else:
i += 1
nc = len(fm0['components'][()])
ns = len(fm0['stations'][()])
S.stations = fm0['stations'][()].astype('U').tolist()
S.components = fm0['components'][()].astype('U').tolist()
S.latitude = T.metadata['latitude']
S.longitude = T.metadata['longitude']
S.depth = T.metadata['depth']
S.template_id = template_id
if ns != 0 and Nsamp != 0:
break
except IndexError:
print("None multiplet for template {:d} !! Return None".format(template_id))
return None
#----------------------------------------------
catalog = autodet.db_h5py.read_catalog_multiplets('multiplets{:d}'.format(template_id), db_path_M=db_path_M, db_path=db_path)
CC = catalog['correlation_coefficients']
best_detection_indexes = np.argsort(CC)[::-1]
if CC.size > 300:
best_detection_indexes = best_detection_indexes[:100] # the best 100 detections
elif CC.size > 100:
best_detection_indexes = best_detection_indexes[:int(30./100. * CC.size)] # the best 30%
elif CC.size > 50:
best_detection_indexes = best_detection_indexes[:int(50./100. * CC.size)] # the best 50%
elif CC>size > 10:
best_detection_indexes = best_detection_indexes[:int(66./100. * CC.size)] # the best 66%
else:
pass # keep all detections
# reorder by chronological order
best_detection_indexes = best_detection_indexes[np.argsort(catalog['origin_times'][best_detection_indexes])]
# get the waveforms
n_events = best_detection_indexes.size
WAVEFORMS = np.zeros((n_events, ns, nc, Nsamp), dtype=np.float32)
filename0 = db_path + db_path_M + catalog['filenames'][best_detection_indexes[0]].decode('utf-8')
f = h5.File(filename0 + 'wav.h5', mode='r')
for n in range(n_events):
filename = db_path + db_path_M + catalog['filenames'][best_detection_indexes[n]].decode('utf-8')
if filename == filename0:
pass
else:
f.close()
f = h5.File(filename + 'wav.h5', mode='r')
WAVEFORMS[n, :, :, :] = f['waveforms'][catalog['indices'][best_detection_indexes[n]], :, :, :]
# normalization
for s in range(ns):
for c in range(nc):
if normRMS:
norm = np.std(WAVEFORMS[n, s, c, :])
else:
norm = np.abs(WAVEFORMS[n, s, c, :]).max()
if norm != 0.:
WAVEFORMS[n, s, c, :] /= norm
filtered_data = np.zeros((n_events, ns, nc, Nsamp), dtype=np.float32)
for s in range(ns):
for c in range(nc):
filtered_data[:,s,c,:] = SVDWF(WAVEFORMS[:,s,c,:], Nsing_values, max_freq=max_freq)
#filtered_data[:,s,c,:] = spectral_filtering(WAVEFORMS[:,s,c,:], SNR_thres=5., max_freq=max_freq)
mean = np.mean(filtered_data[:,s,c,:], axis=0)
mean /= np.abs(mean).max()
S += Trace(data=mean)
S[-1].stats.station = S.stations[s]
S[-1].stats.channel = S.components[c]
S[-1].stats.sampling_rate = autodet.cfg.sampling_rate
S.data = filtered_data
if attach_raw_data:
S.raw_data = WAVEFORMS
S.Nmulti = best_detection_indexes.size
return S
