def test_update():
# copy data
os.mkdir('test/testdata/testsrc/step_0/corr')
os.mkdir('test/testdata/testsrc/step_0/grad')
os.system(
'cp test/testdata/testsrc/step_0/corr_archived/NET.STA1..CHA--NET.STA2..CHA.sac \
test/testdata/testsrc/step_0/corr/NET.STA1..CHA--NET.STA2..CHA.sac')
os.system('cp test/testdata/testsrc/step_0/grad_archived/grad_all.npy\
test/testdata/testsrc/step_0/grad/grad_all.npy')
os.system('cp test/testdata/testsrc/step_0/starting_model_archived.h5\
test/testdata/testsrc/step_0/starting_model.h5')
os.system(
'cp test/testdata/testsrc/step_0/ln_energy_ratio.measurement_archived.csv\
test/testdata/testsrc/step_0/ln_energy_ratio.0.measurement.csv')
# run forward model
os.system('./test/testdata/testsrc/update.sh')
# assert the results are the same
# ToDo: path
n1 = NoiseSource('test/testdata/testsrc/step_1_archived/starting_model.h5')
n2 = NoiseSource('test/testdata/testsrc/step_1/starting_model.h5')
assert (n1.distr_basis == n2.distr_basis).sum() == len(
n1.distr_basis[0, :])
# remove stuff
os.system('rm -rf test/testdata/testsrc/step_0/grad')
os.system('rm -rf test/testdata/testsrc/step_0/corr')
os.system('rm -rf test/testdata/testsrc/step_1')
os.system('rm test/testdata/testsrc/step_0/starting_model.h5')
os.system(
'rm test/testdata/testsrc/step_0/ln_energy_ratio.0.measurement.csv')
def test_sensitivity_kernel():
class args(object):
def __init__(self):
self.source_model = os.path.join('test', 'testdata_v1',
'testsource_v1')
self.step = 0
self.steplengthrun = False,
self.ignore_network = True
args = args()
all_config = config_params(args, comm, size, rank)
ns = get_ns(all_config)
p = define_kernel_tasks(all_config, comm, size, rank)
assert len(p[0]) == 3
assert p[0][2][1].split()[-1] == 'STA2'
input_files = input_files_kernel(p[0][1], all_config)
nsrc = os.path.join('test', 'testdata_v1', 'testsource_v1',
'spectral_model.h5')
output_file = "test"
# use a one-sided taper: The seismogram probably has a non-zero end,
# being cut off wherever the solver stopped running.
taper = cosine_taper(ns[0], p=0.01)
taper[0:ns[0] // 2] = 1.0
kernel = compute_kernel(input_files[0], output_file, all_config,
NoiseSource(nsrc), ns, taper)
np.save("newtestkernel.npy", kernel)
saved_kernel = np.load(
os.path.join('test', 'testdata_v1', 'testdata',
'NET.STA1..MXZ--NET.STA2..MXZ.0.npy'))
assert np.allclose(kernel / kernel.max(),
saved_kernel / saved_kernel.max())
def test_forward_model():
class args(object):
def __init__(self):
self.source_model = os.path.join('test', 'testdata_v1',
'testsource_v1')
self.step = 0
self.steplengthrun = False,
self.ignore_network = True
args = args()
all_config = config_params(args, comm, size, rank)
assert all_config.auto_corr
ns = get_ns(all_config)
assert (ns[0] == 3600)
assert (ns[1] == 7200)
p = define_correlationpairs(all_config.source_config['project_path'],
all_config.auto_corr)
assert len(p) == 3
assert p[0][0].split()[-1] == 'STA1'
input_files = add_input_files(p[1], all_config)[0]
assert os.path.basename(input_files[0]) == 'NET.STA1..MXZ.h5'
nsrc = os.path.join('test', 'testdata_v1', 'testsource_v1', 'iteration_0',
'starting_model.h5')
# use a one-sided taper: The seismogram probably has a non-zero end,
# being cut off wherever the solver stopped running.
taper = cosine_taper(ns[0], p=0.01)
taper[0:ns[0] // 2] = 1.0
correlation, sta1, sta2 = compute_correlation(input_files, all_config,
NoiseSource(nsrc), ns, taper)
corr_saved = np.load(
os.path.join('test', 'testdata_v1', 'testdata',
'NET.STA1..MXZ--NET.STA2..MXZ.npy'))
assert np.allclose(correlation, corr_saved)
def g1g2_kern(wf1str, wf2str, kernel, adjt, src, source_conf, insta):
measr_conf = json.load(
open(os.path.join(source_conf['source_path'], 'measr_config.json')))
bandpass = measr_conf['bandpass']
if bandpass == None:
filtcnt = 1
elif type(bandpass) == list:
if type(bandpass[0]) != list:
filtcnt = 1
else:
filtcnt = len(bandpass)
ntime, n, n_corr, Fs = get_ns(wf1str, source_conf, insta)
# use a one-sided taper: The seismogram probably has a non-zero end,
# being cut off whereever the solver stopped running.
taper = cosine_taper(ntime, p=0.01)
taper[0:ntime // 2] = 1.0
########################################################################
# Prepare filenames and adjoint sources
########################################################################
filenames = []
adjt_srcs = []
adjt_srcs_cnt = 0
for ix_f in range(filtcnt):
filename = kernel + '.{}.npy'.format(ix_f)
filenames.append(filename)
#if os.path.exists(filename):
# continue
f = Stream()
for a in adjt:
adjtfile = a + '*.{}.sac'.format(ix_f)
adjtfile = glob(adjtfile)
try:
f += read(adjtfile[0])[0]
f[-1].data = my_centered(f[-1].data, n_corr)
adjt_srcs_cnt += 1
except IndexError:
print('No adjoint source found: {}\n'.format(a))
break
adjt_srcs.append(f)
########################################################################
# Compute the kernels
########################################################################
with NoiseSource(src) as nsrc:
ntraces = nsrc.src_loc[0].shape[0]
if insta:
# open database
dbpath = json.load(
open(os.path.join(source_conf['project_path'],
'config.json')))['wavefield_path']
# open and determine Fs, nt
db = instaseis.open_db(dbpath)
# get receiver locations
lat1 = geograph_to_geocent(float(wf1[2]))
lon1 = float(wf1[3])
rec1 = instaseis.Receiver(latitude=lat1, longitude=lon1)
lat2 = geograph_to_geocent(float(wf2[2]))
lon2 = float(wf2[3])
rec2 = instaseis.Receiver(latitude=lat2, longitude=lon2)
else:
wf1 = WaveField(wf1str)
wf2 = WaveField(wf2str)
kern = np.zeros((filtcnt, ntraces, len(adjt)))
########################################################################
# Loop over locations
########################################################################
for i in range(ntraces):
# noise source spectrum at this location
# For the kernel, this contains only the basis functions of the
# spectrum without weights; might still be location-dependent,
# for example when constraining sensivity to ocean
S = nsrc.get_spect(i)
if S.sum() == 0.:
# The spectrum has 0 phase so only checking absolute value here
continue
####################################################################
# Get synthetics
####################################################################
if insta:
# get source locations
lat_src = geograph_to_geocent(nsrc.src_loc[1, i])
lon_src = nsrc.src_loc[0, i]
fsrc = instaseis.ForceSource(latitude=lat_src,
longitude=lon_src,
f_r=1.e12)
s1 = np.ascontiguousarray(
db.get_seismograms(
source=fsrc,
receiver=rec1,
dt=1. / source_conf['sampling_rate'])[0].data * taper)
s2 = np.ascontiguousarray(
db.get_seismograms(
source=fsrc,
receiver=rec2,
dt=1. / source_conf['sampling_rate'])[0].data * taper)
else:
s1 = np.ascontiguousarray(wf1.data[i, :] * taper)
s2 = np.ascontiguousarray(wf2.data[i, :] * taper)
spec1 = np.fft.rfft(s1, n)
spec2 = np.fft.rfft(s2, n)
g1g2_tr = np.multiply(np.conjugate(spec1), spec2)
c = np.multiply(g1g2_tr, S)
#######################################################################
# Get Kernel at that location
#######################################################################
corr_temp = my_centered(np.fft.ifftshift(np.fft.irfft(c, n)),
n_corr)
#######################################################################
# Apply the 'adjoint source'
#######################################################################
for ix_f in range(filtcnt):
f = adjt_srcs[ix_f]
if f == None:
continue
for j in range(len(f)):
delta = f[j].stats.delta
kern[ix_f, i, j] = np.dot(corr_temp, f[j].data) * delta
#elif measr_conf['mtype'] in ['envelope']:
# if j == 0:
# corr_temp_h = corr_temp
# print(corr_temp_h)
# if j == 1:
# corr_temp_h = hilbert(corr_temp)
# print(corr_temp_h)
#
# kern[ix_f,i,j] = np.dot(corr_temp,f[j].data) * delta
if i % 50000 == 0:
print("Finished {} source locations.".format(i))
if not insta:
wf1.file.close()
wf2.file.close()
for ix_f in range(filtcnt):
filename = filenames[ix_f]
if kern[ix_f, :, :].sum() != 0:
np.save(filename, kern[ix_f, :, :])
return ()
def run_corr(args, comm, size, rank):
all_conf = config_params(args, comm, size, rank)
# Distributing the tasks
correlation_tasks, n_p_p, n_p = define_correlation_tasks(all_conf,
comm, size, rank)
if len(correlation_tasks) == 0:
return()
if all_conf.config['verbose']:
print('Rank number %g' % rank)
print('working on pair nr. %g to %g of %g.' % (rank * n_p_p,
rank * n_p_p +
n_p_p, n_p))
# Current model for the noise source
it_dir = os.path.join(all_conf.source_config['project_path'],
all_conf.source_config['source_name'],
'iteration_' + str(all_conf.step))
nsrc = os.path.join(it_dir,'starting_model.h5')
# Smart numbers
all_ns = get_ns(all_conf) # ntime, n, n_corr, Fs
# use a one-sided taper: The seismogram probably has a non-zero end,
# being cut off wherever the solver stopped running.
taper = cosine_taper(all_ns[0], p=0.01)
taper[0: all_ns[0] // 2] = 1.0
with NoiseSource(nsrc) as nsrc:
for cp in correlation_tasks:
try:
input_files_list = add_input_files(cp, all_conf)
output_files = add_output_files(cp, all_conf)
except (IndexError, FileNotFoundError):
if all_conf.config['verbose']:
print('Could not determine correlation for: %s\
\nCheck if wavefield .h5 file is available.' % cp)
continue
if type(input_files_list[0]) != list:
input_files_list = [input_files_list]
for i, input_files in enumerate(input_files_list):
correlation, sta1, sta2 = compute_correlation(input_files,
all_conf, nsrc,
all_ns, taper)
add_metadata_and_write(correlation, sta1, sta2,
output_files[i], all_ns[3])
if all_conf.source_config["rotate_horizontal_components"]:
fls = glob(os.path.join(it_dir, "corr", "*{}*{}*.sac".format(cp[0].split()[1],
cp[1].split()[1])))
fls.sort()
apply_rotation(fls,
stationlistfile=os.path.join(all_conf.source_config['project_path'],
"stationlist.csv"), output_directory=os.path.join(it_dir, "corr"))
comm.barrier()
if rank == 0:
if all_conf.source_config["rotate_horizontal_components"]:
fls_to_remove = glob(os.path.join(it_dir, "corr", "*MX[E,N]*MX[E,N]*.sac"))
fls_to_remove.extend(glob(os.path.join(it_dir, "corr", "*MX[E,N]*MXZ*.sac")))
fls_to_remove.extend(glob(os.path.join(it_dir, "corr", "*MXZ*MX[E,N]*.sac")))
for f in fls_to_remove:
os.system("rm " + f)
return()
'testsource_v1')
self.step = 0
self.steplengthrun = False,
self.ignore_network = True
args = args()
all_config = config_params(args, comm, size, rank)
m_a_options = {'g_speed': g_speed, 'window_params': window_params}
m_func = rm.get_measure_func(mtype)
all_ns = get_ns(all_config)
ntime, n, n_corr, Fs = all_ns
taper = cosine_taper(ntime, p=0.01)
taper[0:ntime // 2] = 1.0
with NoiseSource(source_file) as n:
surf_area = n.surf_area
# sum the dimension that relates to filters
grad = grad.sum(axis=1)
# apply surface elements for integration
for j in range(grad.shape[0]):
grad[j, :, 0] *= surf_area
# measurement
obs = read(obsfile)[0]
syn = read(synfile)[0]
syn.stats.sac = {}
syn.stats.sac['dist'] = obs.stats.sac['dist']
msr_o = m_func(obs, **m_a_options)
def run_kern(args, comm, size, rank):
args.steplengthrun = False # by default
all_conf = config_params(args, comm, size, rank)
kernel_tasks, n_p_p, n_p = define_kernel_tasks(all_conf, comm, size, rank)
if len(kernel_tasks) == 0:
return ()
if all_conf.config['verbose']:
print('Rank number %g' % rank)
print('working on pair nr. %g to %g of %g.' %
(rank * n_p_p, rank * n_p_p + n_p_p, n_p))
# Current model for the noise source
nsrc = os.path.join(all_conf.source_config['project_path'],
all_conf.source_config['source_name'],
'spectral_model.h5')
# Smart numbers
all_ns = get_ns(all_conf) # ntime, n, n_corr, Fs
# use a one-sided taper: The seismogram probably has a non-zero end,
# being cut off wherever the solver stopped running.
taper = cosine_taper(all_ns[0], p=0.01)
taper[0:all_ns[0] // 2] = 1.0
with NoiseSource(nsrc) as nsrc:
for kp in kernel_tasks:
try:
input_file_list = add_input_files(kp, all_conf)
output_files = add_output_files(kp, all_conf)
except (IOError, IndexError):
if all_conf.config['verbose']:
print('Could not find input for: %s\
\nCheck if wavefield .h5 file and base_model file are available.' % kp)
continue
for i, input_files in enumerate(input_file_list):
kern = compute_kernel(input_files, output_files[i], all_conf,
nsrc, all_ns, taper)
if kern is None:
continue
for ix_f in range(all_conf.filtcnt):
if not all_conf.source_config[
"rotate_horizontal_components"]:
if kern[:, ix_f, :, :].sum() != 0:
filename = output_files[i] + '.{}.npy'.format(ix_f)
np.save(filename, kern[:, ix_f, :, :])
else:
continue
# Rotation
if all_conf.source_config["rotate_horizontal_components"]:
for kp in kernel_tasks:
try:
input_file_list = add_input_files(kp, all_conf)
output_files = add_output_files(kp, all_conf)
except (IOError, IndexError):
continue
output_files = [
re.sub("MXE", "MXT", of) for of in output_files
]
output_files = [
re.sub("MXN", "MXR", of) for of in output_files
]
# - get all the adjoint sources:
# for all channels, all filter bands, both branches
adjt_srcs = []
for infile in input_file_list:
adjt_srcs.append(
open_adjoint_sources(all_conf, infile[2], all_ns[2]))
# - open 9 temp files
if all_conf.source_config["rotate_horizontal_components"]:
it_dir = os.path.join(
all_conf.source_config['project_path'],
all_conf.source_config['source_name'],
'iteration_' + str(all_conf.step))
tfname = os.path.join(
it_dir, "kern",
"*{}*{}*_temp".format(kp[0].split()[1].strip(),
kp[1].split()[1].strip()))
kern_temp_files = glob(tfname)
if kern_temp_files == []:
continue
kern_temp_files.sort()
if len(kern_temp_files) == 9:
assemble_rotated_kernel(
kern_temp_files, output_files, adjt_srcs,
os.path.join(
all_conf.source_config["project_path"],
"stationlist.csv"))
return ()
# =============================================================================
# initialize stuff
comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()
global itargs
itargs = iteration_args()
if do_not_print_warnings:
warnings.filterwarnings("ignore")
# read the starting model
model_file_name = os.path.join(itargs.iteration, 'starting_model.h5')
os.system('cp ' + model_file_name + ' starting_model_iteration_0.h5')
with NoiseSource(model_file_name) as n0:
# rearrange as 1-D array
model_shape = n0.distr_basis.shape
print("model shape ", model_shape)
x0 = n0.distr_basis[:].ravel()
x_ref = x0.copy()
# call the optimization
result = minimize(objective_function,
x0=x0,
args=(model_shape, weights, regularization, smoothing_params,
x_ref, comm, size, rank, itargs),
method=optimization_scheme,
bounds=bounds,
jac=jacobian,
tol=tol,
def g1g2_kern(wf1, wf2, corr_file, kernel, adjt, src, source_conf, scale=1.0):
#ToDo: Take care of saving metainformation
#ToDo: Think about how to manage different types of sources (numpy array vs. get from configuration -- i.e. read source from file as option)
#ToDo: check whether to include autocorrs from user (now hardcoded off)
#ToDo: Parallel loop(s)
#ToDo tests
ntime, n, n_corr = get_ns(wf1, source_conf)
taper = cosine_taper(ntime, p=0.05)
with WaveField(wf1) as wf1, WaveField(wf2) as wf2:
if wf1.stats['Fs'] != wf2.stats['Fs']:
msg = 'Sampling rates of synthetic green\'s functions must match.'
raise ValueError(msg)
# initialize new hdf5 files for correlation and green's function correlation
#with wf1.copy_setup(corr_file,nt=n_corr) as correl, NoiseSource(src) as nsrc:
#with wf1.copy_setup(corr_file,nt=n_corr) as correl:
with NoiseSource(src) as nsrc:
correlation = np.zeros(n_corr)
kern = np.zeros(wf1.stats['ntraces'])
# Try to use a trick: Use causal and acausal part of f separately.
f = read(adjt)[0]
f.data = my_centered(f.data, n_corr)
n_acausal_samples = (f.stats.npts - 1) / 2
specf = np.fft.rfft(f[n_acausal_samples:], n)
# Loop over source locations
#with click.progressbar(range(wf1.stats['ntraces']),\
#label='Correlating...' ) as ind:
for i in range(wf1.stats['ntraces']):
#print(i)
s1 = np.ascontiguousarray(wf1.data[i, :] * taper) * scale
#print(s1.sum())
spec1 = np.fft.rfft(s1, n)
#print(spec1.sum())
T = np.multiply(np.conj(spec1), nsrc.get_spect(i))
# plt.plot(np.abs(spec1)/np.max(np.abs(spec1)))
# plt.plot(np.abs(T)/np.max(np.abs(T)))
# plt.show()
# it would be cleaner to use ifftshift here!
T = np.fft.ifftshift(np.fft.irfft(T,
n))[0:len(s1)] #[-len(s1):]
# if i in [1,2,3,4,5]:
# plt.plot(T[::-1]/np.max(np.abs(T)))
# plt.plot(s1/np.max(np.abs(s1)),'--')
# plt.show()
# Get s2 in the shape of f
# we need to add half of the length of f before G to replace the
# acausal part that G does not have to start with:
#s2 = np.zeros(n)
s2 = np.ascontiguousarray(wf2.data[i, :] * taper) * scale
#print(s2.sum())
#s2[n_acausal_samples:n_acausal_samples+ntime] = s2_caus
spec2 = np.fft.rfft(s2, n)
# plt.plot(s2_caus/np.max(np.abs(s2_caus)))
# plt.plot(f.data/np.max(np.abs(f.data)))
# plt.plot(s2/np.max(np.abs(s2)))
# plt.plot(n_acausal_samples,0.5,'rd')
# plt.show()
# transform both f and s2 to fourier d
# (again, zeropadding but just to avoid circular convolution)
# plt.plot(np.abs(spec2)/np.max(np.abs(spec2)))
# plt.plot(np.abs(specf)/np.max(np.abs(specf)))
# plt.show()
g2f_tr = np.multiply(np.conj(spec2), specf)
#print(specf.sum())
#plt.plot(n_acausal_samples,0.5,'rd')
#plt.plot(n,0.5,'gd')
# it would be cleaner to use ifftshift here!
u_dagger = np.fft.ifftshift(np.fft.irfft(
g2f_tr, n))[0:len(s1)] #[-len(s1):]
# plt.plot(u_dagger/np.max(np.abs(u_dagger)))
# plt.plot(T/np.max(np.abs(T)))
# plt.show()
# The frequency spectrum of the noise source is included here
## corr_temp = my_centered(np.fft.ifftshift(np.fft.irfft(c,n)),n_corr)
# A Riemann sum -- one could actually build in a more fancy integration here
#print(f.stats.delta)
kern[i] = np.dot(u_dagger, T) * f.stats.delta
#print(kern[i])
if i % 50000 == 0:
print("Finished {} source locations.".format(i))
#np.save(kernel,kern)
return (kern)
def g1g2_corr(wf1, wf2, corr_file, kernel, adjt, src, source_conf, kernelrun):
#ToDo: Take care of saving metainformation
#ToDo: Think about how to manage different types of sources (numpy array vs. get from configuration -- i.e. read source from file as option)
#ToDo: check whether to include autocorrs from user (now hardcoded off)
#ToDo: Parallel loop(s)
#ToDo tests
ntime, n, n_corr = get_ns(wf1, source_conf)
taper = cosine_taper(ntime, p=0.05)
with WaveField(wf1) as wf1, WaveField(wf2) as wf2:
if wf1.stats['Fs'] != wf2.stats['Fs']:
msg = 'Sampling rates of synthetic green\'s functions must match.'
raise ValueError(msg)
# initialize new hdf5 files for correlation and green's function correlation
#with wf1.copy_setup(corr_file,nt=n_corr) as correl, NoiseSource(src) as nsrc:
#with wf1.copy_setup(corr_file,nt=n_corr) as correl:
with NoiseSource(src) as nsrc:
correlation = np.zeros(n_corr)
if kernelrun:
#if not os.path.exists(adjt):
# print('Adjoint source %s not found, skipping kernel.')
# return()
kern = np.zeros((wf1.stats['ntraces'], len(adjt)))
f = Stream()
for adjtfile in adjt:
if adjtfile == '-':
return
f += read(adjtfile)[0]
f[-1].data = my_centered(f[-1].data, n_corr)
# Loop over source locations
#with click.progressbar(range(wf1.stats['ntraces']),\
#label='Correlating...' ) as ind:
for i in range(wf1.stats['ntraces']):
# noise source spectrum at this location
# if calculating kernel, the spectrum is location independent.
S = nsrc.get_spect(i)
if S.sum() == 0.: # The spectrum has 0 phase anyway
continue
s1 = np.ascontiguousarray(wf1.data[i, :] * taper)
s2 = np.ascontiguousarray(wf2.data[i, :] * taper)
spec1 = np.fft.rfft(s1, n)
spec2 = np.fft.rfft(s2, n)
g1g2_tr = np.multiply(np.conjugate(spec1), spec2)
# if i%50000 == 0:
# print(g1g2_tr[0:10],file=None)
# print(g1g2_tr.max(),file=None)
c = np.multiply(g1g2_tr, S)
# if i%50000==0:
# print(c[0:10],file=None)
# print(c.max(),file=None)
if kernelrun:
corr_temp = my_centered(
np.fft.ifftshift(np.fft.irfft(c, n)), n_corr)
##if i%50000 == 0:
# ##print(corr_temp[0:10],file=None)
#print(corr_temp.max(),file=None)
# A Riemann sum
for j in range(len(adjt)):
kern[i, j] = np.dot(corr_temp,
f[j].data) * f[j].stats.delta
else:
correlation += my_centered(
np.fft.ifftshift(np.fft.irfft(c, n)), n_corr)
if i % 50000 == 0:
print("Finished {} source locations.".format(i))
if kernelrun:
np.save(kernel, kern)
else:
trace = Trace()
trace.stats.sampling_rate = wf1.stats['Fs']
trace.data = correlation
trace.write(filename=corr_file, format='SAC')
def assemble_ascent_dir(source_model,step,snr_min,n_min,save_all=False,
normalize_gradient=False):
# where is the measurement database located?
source_config=json.load(open(source_model))
datadir = os.path.join(source_config['source_path'],'step_' + str(step))
outfile = os.path.join(datadir,'grad','grad_info.txt')
if os.path.exists(outfile):
os.system('rm '+outfile)
# Figure out how many spectral basis functions there are:
with NoiseSource(os.path.join(datadir,'starting_model.h5')) as nsrc:
n_basis = nsrc.spect_basis.shape[0]
# allocate the kernel array
grd = np.load(os.path.join(source_config['project_path'],'sourcegrid.npy'))
gradient = np.zeros((n_basis,np.shape(grd)[1]))
# get the predefined weights
measr_config = json.load(open(os.path.join(source_config['source_path'],\
'measr_config.json')))
m_type = measr_config['mtype']
try:
var_weights = measr_config['weights']
except KeyError:
var_weights = np.ones(n_basis)
# Loop over basis functions
for ix_basis in range(n_basis):
msrfile = os.path.join(datadir,"{}.{}.measurement.csv".\
format(measr_config['mtype'],ix_basis))
# Read in the csv files of measurement.
data = pd.read_csv(msrfile)
# loop over stationpairs
cnt_success = 0
cnt_lowsnr = 0
cnt_lown = 0
cnt_overlap = 0
cnt_unavail = 0
n = len(data)
print('Nr Measurements:')
print(n)
print('*'*16)
for i in range(n):
if data.at[i,'snr'] < snr_min and data.at[i,'snr_a'] < snr_min:
cnt_lowsnr += 1
continue
if data.at[i,'nstack'] < n_min:
cnt_lown += 1
continue
# ToDo: deal with station pairs with several measurements (with different instruments)
# (At the moment, just all added. Probably fine on this large scale)
# find kernel file
sta1 = data.at[i,'sta1']
sta2 = data.at[i,'sta2']
#if sta1.split('.')[-1][-1] in ['E','N','T','R']:
# msg = "Cannot yet handle horizontal components"
# raise NotImplementedError(msg)
# if sta2.split('.')[-1][-1] in ['E','N','T','R']:
# msg = "Cannot yet handle horizontal components"
# raise NotImplementedError(msg)
# ToDo !!! Replace this by a decent formulation, where the channel is properly set !!! No error for E, R, T, N
sta1 = "*.{}..{}".format(sta1.split('.')[1],source_config['channel']) # ignoring network: IRIS has sometimes several network codes at same station
sta2 = "*.{}..{}".format(sta2.split('.')[1],source_config['channel']) # ignoring network: IRIS has sometimes several network codes at same station
kernelfile1 = os.path.join(datadir,'kern',"{}--{}.{}.npy".format(sta1,sta2,ix_basis))
kernelfile2 = os.path.join(datadir,'kern',"{}--{}.{}.npy".format(sta2,sta1,ix_basis))
# Same problem with different network codes.
# Due to station pairs being in alphabetic order of network.station.loc.cha, different network
# codes also lead to different ordering.
try:
kernelfile = glob(kernelfile1)[0]
except IndexError:
try:
kernelfile = glob(kernelfile2)[0]
except IndexError:
kernelfile = kernelfile1
# Check that first, and then complain.
# Skip if entry is nan: This is most likely due to no measurement taken because station distance too short
if (isnan(data.at[i,'obs']) and m_type
in ['ln_energy_ratio','energy_diff']):
print("No measurement in dataset for:")
print(os.path.basename(kernelfile))
cnt_overlap += 1
continue
# ...unless somehow the kernel went missing (undesirable case!)
if not os.path.exists(kernelfile):
print("File does not exist:")
print(os.path.basename(kernelfile))
cnt_unavail += 1
continue
# load kernel
kernel = np.load(kernelfile)
if True in np.isnan(kernel):
print("kernel contains nan, skipping")
print(os.path.basename(kernelfile))
continue
# multiply kernel and measurement, add to descent dir.
# always assuming L2 norm here!
else:
if kernel.shape[-1] == 1:
kernel = kernel[:,0]
if m_type in ['ln_energy_ratio','energy_diff']:
kernel *= (data.at[i,'syn'] - data.at[i,'obs'])
elif kernel.shape[-1] == 2:
if m_type in ['ln_energy_ratio','energy_diff']:
kernel[:,0] *= (data.at[i,'syn'] - data.at[i,'obs'])
kernel[:,1] *= (data.at[i,'syn_a'] - data.at[i,'obs_a'])
kernel = kernel[:,0] + kernel[:,1]
#if m_type in ['envelope']:
# kernel = kernel[:,0] + kernel[:,1]
cnt_success += 1 # yuhu
# collect
gradient[ix_basis,:] += kernel * var_weights[ix_basis]
del kernel
# save
if save_all:
warn('This option is discontinued, because all the single kernels are\
available in the kern/ directory.')
if normalize_gradient:
gradient /= np.abs(gradient).max()
kernelfile = os.path.join(datadir,'grad','grad_all.npy')
np.save(kernelfile,gradient)
# output metadata
with open(outfile,'a') as fh:
fh.write('Analyzed %g station pairs of %g successfully.\n' %(cnt_success,n))
fh.write('No data found for %g station pairs.\n' %cnt_unavail)
fh.write('No measurement taken for %g station pairs due to short interstation distance.\n' %cnt_overlap)
fh.write('Signal to noise ratio below threshold for %g station pairs.\n' %cnt_lowsnr)
fh.write('Number of staacked windows below threshold for %g station pairs.\n' %cnt_lown)
fh.write('\nParameters:==============================================================\n')
fh.write('Source dir: %s \n' %source_model)
fh.write('Step: %g' %int(step))
fh.write('Minimum SNR: %g' %snr_min)
fh.write('Minimum stack length: %g' %int(n_min))
fh.write('Save all interstation gradients: %s' %str(save_all))
fh.write('\n=========================================================================\n')
fh.write('Project:\n')
# append configurations
cfg = open(os.path.join(source_config['project_path'],'config.json')).read()
fh.write(cfg)
fh.write('\n=========================================================================\n')
fh.write('Source model:\n')
fh.write(json.dumps(source_config))
fh.write('\n=========================================================================\n')
fh.write('Measurement:\n')
cfg = open(os.path.join(source_config['source_path'],'measr_config.json')).read()
fh.write(cfg)
def g1g2_corr(wf1, wf2, corr_file, src, source_conf, insta):
"""
Compute noise cross-correlations from two .h5 'wavefield' files.
Noise source distribution and spectrum is given by starting_model.h5
It is assumed that noise sources are delta-correlated in space.
"""
#ToDo: check whether to include autocorrs from user (now hardcoded off)
#ToDo: Parallel loop(s)
#ToDo tests
# Metainformation: Include the reference station names for both stations
# from wavefield files, if possible. Do not include geographic information
# from .csv file as this might be error-prone. Just add the geographic
# info later if needed.
with NoiseSource(src) as nsrc:
ntime, n, n_corr, Fs = get_ns(wf1, source_conf, insta)
# use a one-sided taper: The seismogram probably has a non-zero end,
# being cut off whereever the solver stopped running.
taper = cosine_taper(ntime, p=0.01)
taper[0:ntime // 2] = 1.0
ntraces = nsrc.src_loc[0].shape[0]
print(taper.shape)
correlation = np.zeros(n_corr)
if insta:
# open database
dbpath = json.load(
open(os.path.join(source_conf['project_path'],
'config.json')))['wavefield_path']
# open and determine Fs, nt
db = instaseis.open_db(dbpath)
# get receiver locations
lat1 = geograph_to_geocent(float(wf1[2]))
lon1 = float(wf1[3])
rec1 = instaseis.Receiver(latitude=lat1, longitude=lon1)
lat2 = geograph_to_geocent(float(wf2[2]))
lon2 = float(wf2[3])
rec2 = instaseis.Receiver(latitude=lat2, longitude=lon2)
else:
wf1 = WaveField(wf1)
wf2 = WaveField(wf2)
# Loop over source locations
for i in range(ntraces):
# noise source spectrum at this location
S = nsrc.get_spect(i)
if S.sum() == 0.:
#If amplitude is 0, continue. (Spectrum has 0 phase anyway. )
continue
if insta:
# get source locations
lat_src = geograph_to_geocent(nsrc.src_loc[1, i])
lon_src = nsrc.src_loc[0, i]
fsrc = instaseis.ForceSource(latitude=lat_src,
longitude=lon_src,
f_r=1.e12)
s1 = np.ascontiguousarray(
db.get_seismograms(
source=fsrc,
receiver=rec1,
dt=1. / source_conf['sampling_rate'])[0].data * taper)
s2 = np.ascontiguousarray(
db.get_seismograms(
source=fsrc,
receiver=rec2,
dt=1. / source_conf['sampling_rate'])[0].data * taper)
else:
# read Green's functions
s1 = np.ascontiguousarray(wf1.data[i, :] * taper)
s2 = np.ascontiguousarray(wf2.data[i, :] * taper)
# Fourier transform for greater ease of convolution
spec1 = np.fft.rfft(s1, n)
spec2 = np.fft.rfft(s2, n)
# convolve G1G2
g1g2_tr = np.multiply(np.conjugate(spec1), spec2)
# convolve noise source
c = np.multiply(g1g2_tr, S)
# transform back
correlation += my_centered(np.fft.ifftshift(np.fft.irfft(c, n)),
n_corr) * nsrc.surf_area[i]
# occasional info
if i % 50000 == 0:
print("Finished {} source locations.".format(i))
###################### end of loop over all source locations ###################
if not insta:
wf1.file.close()
wf2.file.close()
# save output
trace = Trace()
trace.stats.sampling_rate = Fs
trace.data = correlation
# try to add some meta data
try:
sta1 = wf1.stats['reference_station']
sta2 = wf2.stats['reference_station']
trace.stats.station = sta1.split('.')[1]
trace.stats.network = sta1.split('.')[0]
trace.stats.location = sta1.split('.')[2]
trace.stats.channel = sta1.split('.')[3]
trace.stats.sac = {}
trace.stats.sac['kuser0'] = sta2.split('.')[1]
trace.stats.sac['kuser1'] = sta2.split('.')[0]
trace.stats.sac['kuser2'] = sta2.split('.')[2]
trace.stats.sac['kevnm'] = sta2.split('.')[3]
except:
pass
trace.write(filename=corr_file, format='SAC')