def test_regularization(self):
x = np.linspace(0, 10, 4)
model_true = np.array([2.5, 5.0])
data = f(model_true, x)
model_pred = modest.nonlin_lstsq(f, data, 2, regularization=(0, 0.0), system_args=(x,))
self.assertTrue(np.linalg.norm(model_pred - model_true) < tol)
model_pred = modest.nonlin_lstsq(f, data, 2, regularization=(0, 1e8), system_args=(x,))
self.assertTrue(np.linalg.norm(model_pred) < tol)
model_pred = modest.nonlin_lstsq(f, data, 2, regularization=(1, 1e8), system_args=(x,))
self.assertTrue((model_pred[0] - model_pred[1]) < tol)
def test_solver_nnls(self):
x = np.linspace(0, 10, 4)
# the bounds should not impede the results here
model_true = np.array([0.1, 5.0])
data = f_nonlin(model_true, x)
model_pred = modest.nonlin_lstsq(f_nonlin, data, 2, system_args=(x,), solver=modest.nnls)
self.assertTrue(np.linalg.norm(model_pred - model_true) < tol)
self.assertTrue(all(model_pred >= 0.0))
# the bounds should influence the results here
model_true = np.array([-2.5, 5.0])
data = f_nonlin(model_true, x)
model_pred = modest.nonlin_lstsq(f_nonlin, data, 2, system_args=(x,), solver=modest.nnls)
self.assertTrue(all(model_pred >= 0.0))
def test_nonnegativity(self):
soln0 = modest.nonlin_lstsq(system,
data,
model_prior,
data_covariance=data_cov,
prior_covariance=model_prior_cov,
system_args=(time,),
solver=modest.nnls)
kf1 = modest.KalmanFilter(model_prior,
model_prior_cov,
system_kf,
solver=modest.nonlin_lstsq_update,
solver_kwargs={'solver':modest.nnls})
kf1.filter(data_kf,data_cov_kf,time)
soln1 = kf1.get_posterior()[0]
self.assertTrue(np.all(np.isclose(soln0,soln1)))
def test_bayes_least_squares(self):
soln1,cov1 = modest.nonlin_lstsq(system,
data,
model_prior,
data_covariance=data_cov,
prior_covariance=model_prior_cov,
system_args=(time,),
output=['solution','solution_covariance'])
kf = modest.KalmanFilter(model_prior,
model_prior_cov,
system_kf)
kf.filter(data_kf,data_cov_kf,time)
soln2,cov2 = kf.get_posterior()
kf.close()
self.assertTrue(np.all(np.isclose(soln1,soln2)))
self.assertTrue(np.all(np.isclose(cov1,cov2)))
def test_smoothing_no_core(self):
pred1 = modest.nonlin_lstsq(system,
data,
model_prior,
data_covariance=data_cov,
prior_covariance=model_prior_cov,
system_args=(time,),
output=['predicted'])
kf = modest.KalmanFilter(model_prior,
model_prior_cov,
system_kf,
core=False)
kf.filter(data_kf,data_cov_kf,time,smooth=True)
pred2 = np.array([system_kf(kf.history['smooth'][i,:],t) for i,t in enumerate(time)])
kf.close()
self.assertTrue(np.all(np.isclose(pred1[:,None],pred2)))
def ECEF_to_geodetic(x,y,z,ref='WGS84'):
'''finds the geodetic coordinates from cartesian coordinates
This is done by solving a nonlinear inverse problem using
'geodetic_to_cartesian' as the forward problem. This is not the
most efficient algorithm.
change to ecef to geodetic
Parameters
----------
x,y,z: (scalars) coordinates in meters
ref: (optional) reference ellipoid. Either 'NAD27','GRS80', or
'WGS84'
Returns
-------
lon,lat,h: longitude (degrees), latitude (degrees) and height
above the reference ellipsoid (meters)
The coordinate system is as defined by the WGS84, where the z axis
is pointing in the direction of the IERS Reference pole; the x axis
is the intersection between the equatorial plane and the IERS
Reference Meridian; and the y axis completes a right-handed
coordinate coordinate system
'''
lon,lat,h = modest.nonlin_lstsq(
_system,
np.array([x,y,z]),
np.zeros(3),
system_args=(ref,),
rtol=1e-10,
atol=1e-10,
maxitr=100,
LM_damping=True,
LM_param=1.0)
#print(lon,lat)
lon,lat = bound_lon_lat(lon,lat)
return lon,lat,h
def ECEF_to_geodetic(x, y, z, ref='WGS84'):
'''finds the geodetic coordinates from cartesian coordinates
This is done by solving a nonlinear inverse problem using
'geodetic_to_cartesian' as the forward problem. This is not the
most efficient algorithm.
change to ecef to geodetic
Parameters
----------
x,y,z: (scalars) coordinates in meters
ref: (optional) reference ellipoid. Either 'NAD27','GRS80', or
'WGS84'
Returns
-------
lon,lat,h: longitude (degrees), latitude (degrees) and height
above the reference ellipsoid (meters)
The coordinate system is as defined by the WGS84, where the z axis
is pointing in the direction of the IERS Reference pole; the x axis
is the intersection between the equatorial plane and the IERS
Reference Meridian; and the y axis completes a right-handed
coordinate coordinate system
'''
lon, lat, h = modest.nonlin_lstsq(_system,
np.array([x, y, z]),
np.zeros(3),
system_args=(ref, ),
rtol=1e-10,
atol=1e-10,
maxitr=100,
LM_damping=True,
LM_param=1.0)
#print(lon,lat)
lon, lat = bound_lon_lat(lon, lat)
return lon, lat, h
def test_masked_arrays(self):
data_indices = [1,2,3,5,6,7,8,10,12]
mask = np.ones((len(data),1),dtype=bool)
mask[data_indices,:] = False
soln1,cov1 = modest.nonlin_lstsq(system,
data,
model_prior,
data_covariance=data_cov,
prior_covariance=model_prior_cov,
data_indices=data_indices,
system_args=(time,),
output=['solution','solution_covariance'])
kf = modest.KalmanFilter(model_prior,
model_prior_cov,
system_kf)
kf.filter(data_kf,data_cov_kf,time,mask=mask)
soln2,cov2 = kf.get_posterior()
kf.close()
self.assertTrue(np.all(np.isclose(soln1,soln2)))
self.assertTrue(np.all(np.isclose(cov1,cov2)))
def test_reg_bayes_least_squares(self):
soln0 = modest.nonlin_lstsq(system,
data,
model_prior,
data_covariance=data_cov,
prior_covariance=model_prior_cov,
system_args=(time,),
regularization=reg_mat)
kf1 = modest.KalmanFilter(model_prior,
model_prior_cov,
system_reg_kf,
obs_args=(0.25819889*reg_mat,))
kf2 = modest.KalmanFilter(model_prior,
model_prior_cov,
system_kf,
solver=modest.nonlin_lstsq_update,
solver_kwargs={'regularization':0.25819889*reg_mat})
kf1.filter(data_reg_kf,data_reg_cov_kf,time)
soln1 = kf1.get_posterior()[0]
kf2.filter(data_kf,data_cov_kf,time)
soln2 = kf2.get_posterior()[0]
kf1.close()
kf2.close()
self.assertTrue(np.all(np.isclose(soln0,soln1,soln2)))
def main(data,gf,param,outfile):
'''
Nt: number of time steps
Nx: number of positions
Dx: spatial dimensions of coordinates and displacements
Ns: number of slip basis functions per slip direction
Ds: number of slip directions
Nv: number of fluidity basis functions
total: number of state parameters (Ns*Ds + Nv + 2*Nx*Dx)
Parameters
----------
data: \ mask : (Nt,Nx) boolean array
\ mean : (Nt,Nx,Dx) array
\ covariance : (Nt,Nx,Dx,Dx) array
\ metadata \ position : (Nx,Dx) array
time : (Nt,) array
prior: \ mean : (total,) array
\ covariance : (total,total) array
gf: \ elastic : (Ns,Ds,Nx,Dx) array
\ viscoelastic : (Ns,Ds,Dv,Nx,Dx) array
\ metadata \ position : (Nx,Dx) array
reg: \ regularization : (*,total) array
params: user parameter dictionary
Returns
-------
out: \ slip_integral \ mean : (Nt,Ns,Ds) array
\ uncertainty :(Nt,Ns,Ds) array
\ slip \ mean : (Nt,Ns,Ds) array
\ uncertainty : (Nt,Ns,Ds) array
\ slip_derivative \ mean : (Nt,Ns,Ds) array
\ uncertainty : (Nt,Ns,Ds) array
\ fluidity \ mean : (Nv,) array
\ uncertainty : (Nv,) array
\ secular_velocity \ mean : (Nx,Dx) array
\ uncertainty : (Nx,Dx) array
\ baseline_displacement \ mean : (Nx,Dx) array
\ uncertainty : (Nx,Dx) array
'''
F = gf['slip'][...]
G = gf['fluidity'][...]
coseismic_times = np.array(param['coseismic_times'])
afterslip_start_times = param['afterslip_start_times']
afterslip_end_times = param['afterslip_end_times']
afterslip_times = zip(afterslip_start_times,afterslip_end_times)
afterslip_times = np.array(afterslip_times)
time = data['time'][:]
slip_scale = 1.0 # meter
relax_scale = 1.0 # year
time_scale = np.std(time)
time_shift = np.mean(time) # years
disp_scale = 1.0 # meter
# shift is different for each time series
disp_shift = np.mean(data['mean'],0)
# scale greens functions
# F is originally in meters disp per meter slip
F /= disp_scale/slip_scale
# G is originally in meters/year disp per meter slip*fluidity
G /= (disp_scale/time_scale)/(slip_scale/relax_scale)
time -= time_shift
time /= time_scale
coseismic_times -= time_shift
coseismic_times /= time_scale
afterslip_times -= time_shift
afterslip_times /= time_scale
param['initial_slip_variance'] /= slip_scale**2
param['fluidity_variance'] *= relax_scale**2
param['secular_velocity_variance'] /= (disp_scale/time_scale)**2
param['baseline_displacement_variance'] /= disp_scale**2
# define slip functions and slip jacobian here
slip_func,slip_jac = steps_and_ramps(coseismic_times,
afterslip_times)
Nst = len(coseismic_times) + len(afterslip_start_times)
Ns,Ds,Nv,Nx,Dx = np.shape(G)
Nt = len(time)
# check for consistency between input
assert data['mean'].shape == (Nt,Nx,Dx)
assert data['variance'].shape == (Nt,Nx,Dx)
assert F.shape == (Ns,Ds,Nx,Dx)
assert G.shape == (Ns,Ds,Nv,Nx,Dx)
p = state_parser(Nst,Ns,Ds,Nv,Nx,Dx)
if param['solver'] == 'bvls':
solver = modest.bvls
upper_bound = 1e6*np.ones(p['total'])
lower_bound = -1e6*np.ones(p['total'])
# all inferred fluidities will be positive
lower_bound[p['fluidity']] = 0
# all inferred left lateral slip will be positive
left_lateral_indices = np.array(p['slip'][:,:,0],copy=True)
thrust_indices = np.array(p['slip'][:,:,1],copy=True)
upper_bound[left_lateral_indices] = 0.0
solver_args = (lower_bound,upper_bound)
solver_kwargs = {}
if param['solver'] == 'lstsq':
solver = modest.lstsq
solver_args = ()
solver_kwargs = {}
elif param['solver'] == 'lsmr':
solver = modest.lsmr
solver_args = ()
solver_kwargs = {}
elif param['solver'] == 'lgmres':
solver = modest.lgmres
solver_args = ()
solver_kwargs = {'tol':1e-8,'maxiter':1000}
elif param['solver'] == 'dgs':
solver = modest.dgs
solver_args = ()
solver_kwargs = {}
#fprior = prior.create_formatted_prior(param,p,slip_model='parameterized')
Xprior,Cprior = prior.create_prior(param,p,slip_model='parameterized')
reg_matrix = reg.create_regularization(param,p,slip_model='parameterized')
reg_rows = len(reg_matrix)
setup_output_file(outfile,p,
data['name'][...],
data['position'][...],
data['time'][...])
Xprior,Cprior = modest.nonlin_lstsq(
regularization,
np.zeros(reg_rows),
Xprior,
data_covariance=np.eye(reg_rows),
prior_covariance=Cprior,
system_args=(reg_matrix,),
jacobian=regularization_jacobian,
jacobian_args=(reg_matrix,),
maxitr=param['maxitr'],
solver=solver,
solver_args=solver_args,
solver_kwargs=solver_kwargs,
LM_damping=True,
output=['solution','solution_covariance'])
time_indices = range(Nt)
block_time_indices = modest.misc.divide_list(time_indices,param['time_blocks'])
for i in block_time_indices:
outfile['data/mean'][i,...] = data['mean'][i,...]
outfile['data/mask'][i,...] = data['mask'][i,...]
outfile['data/covariance'][i,...] = data['variance'][i,...]
di = data['mean'][i,...]
di -= disp_shift
di /= disp_scale
di_mask = np.array(data['mask'][i,:],dtype=bool)
# expand to three dimensions
di_mask = np.repeat(di_mask[...,None],3,-1)
di = di[~di_mask]
Cdi = data['variance'][i,...]
Cdi /= disp_scale**2
Cdi = Cdi[~di_mask]
Xprior,Cprior = modest.nonlin_lstsq(
observation,
di,
Xprior,
data_covariance=Cdi,
prior_covariance=Cprior,
system_args=(time[i],F,G,p,slip_func,di_mask),
jacobian=observation_jacobian,
jacobian_args=(time[i],F,G,p,slip_func,slip_jac,di_mask),
solver=solver,
solver_args=solver_args,
solver_kwargs=solver_kwargs,
maxitr=param['maxitr'],
LM_damping=True,
LM_param=1.0,
rtol=1e-2,
atol=1e-2,
output=['solution','solution_covariance'])
post_mean_scaled,post_cov_scaled = Xprior,Cprior
post_mean = np.copy(post_mean_scaled)
post_mean[p['baseline_displacement']] *= disp_scale
post_mean[p['baseline_displacement']] += disp_shift
post_mean[p['secular_velocity']] *= (disp_scale/time_scale)
post_mean[p['slip']] *= slip_scale
post_mean[p['fluidity']] /= relax_scale
for i in range(Nt):
outfile['state/all'][i,:] = post_mean
outfile['state/baseline_displacement'][i,...] = post_mean[p['baseline_displacement']]
outfile['state/secular_velocity'][i,...] = post_mean[p['secular_velocity']]
outfile['state/slip'][i,...] = slip_func(post_mean[p['slip']],time[i])
outfile['state/slip_derivative'][i,...] = slip_func(post_mean[p['slip']],time[i],diff=1)
outfile['state/fluidity'][i,...] = post_mean[p['fluidity']]
# compute predicted data
logger.info('computing predicted data')
error = 0.0
count = 0
for i in range(Nt):
predicted = observation_t(post_mean_scaled,
time[i],
F,G,p,slip_func)
# change back to meters
predicted *= disp_scale
predicted += disp_shift
residual = outfile['data/mean'][i,...] - predicted
covariance = outfile['data/covariance'][i,...]
data_mask = np.array(outfile['data/mask'][i,...],dtype=bool)
error += L2(residual,covariance,data_mask)
count += np.sum(~data_mask)
outfile['predicted/mean'][i,...] = predicted
mask = np.zeros(p['total'])
mask[p['secular_velocity']] = 1.0
mask[p['baseline_displacement']] = 1.0
mask_post_mean = post_mean_scaled*mask
tectonic = observation_t(mask_post_mean,
time[i],
F,G,p,
slip_func)
tectonic *= disp_scale
tectonic += disp_shift
outfile['tectonic/mean'][i,...] = tectonic
mask = np.zeros(p['total'])
mask[p['slip']] = 1.0
mask_post_mean = post_mean_scaled*mask
elastic = observation_t(mask_post_mean,
time[i],
F,G,p,
slip_func)
elastic *= disp_scale
outfile['elastic/mean'][i,...] = elastic
visc = (outfile['predicted/mean'][i,...] -
outfile['tectonic/mean'][i,...] -
outfile['elastic/mean'][i,...])
outfile['viscous/mean'][i,...] = visc
logger.info('total RMSE: %s' % np.sqrt(error/count))
return
# true mode we are trying to recover
model_true = np.random.random(2)
model_true[0] *= 0.6
model_true[0] += 1.2
model_true[1] *= 0.3
model_true[1] += 0.1
# create synthetic data
N = 100
time = np.linspace(0.0,5.0,100)
data = system(model_true,time) + np.random.normal(0.0,0.1,N)
data_variance = 0.1*np.ones(N)
soln,soln_cov,pred = modest.nonlin_lstsq(system,data,M,
data_covariance=data_variance,
system_args=(time,),
solver=modest.bvls,
solver_args=(lb,ub),
output=['solution','solution_covariance','predicted'])
# plot misfit surface
trialsx = np.linspace(1.0,2.0,200)
trialsy = np.linspace(0.001,0.5,200)
trialsx,trialsy = np.meshgrid(trialsx,trialsy)
trialsx = trialsx.flatten()
trialsy = trialsy.flatten()
trials = np.array([trialsx,trialsy]).T
misfit = np.array([np.sum((system(m,time) - data)**2/data_variance) for m in trials])
model_best = trials[np.argmin(misfit)]
# model space plot
fig,ax = plt.subplots()
def test_solver_lstsq(self):
x = np.linspace(0, 10, 4)
model_true = np.array([2.5, 5.0])
data = f_nonlin(model_true, x)
model_pred = modest.nonlin_lstsq(f_nonlin, data, 2, system_args=(x,), solver=modest.lstsq)
self.assertTrue(np.linalg.norm(model_pred - model_true) < tol)
def test_linear_with_jacobian(self):
x = np.linspace(0, 10, 4)
model_true = np.array([2.5, 5.0])
data = f(model_true, x)
model_pred = modest.nonlin_lstsq(f, data, np.zeros(2), jacobian=jac, jacobian_args=(x,), system_args=(x,))
self.assertTrue(np.linalg.norm(model_pred - model_true) < tol)
def test_linear_no_init_guess(self):
x = np.linspace(0, 10, 4)
model_true = np.array([2.5, 5.0])
data = f(model_true, x)
model_pred = modest.nonlin_lstsq(f, data, 2, system_args=(x,))
self.assertTrue(np.linalg.norm(model_pred - model_true) < tol)
def logexp_filter(u,var,t,start,jumps,trials=10,diff=0,detrend=False,reg=1e-10):
'''
Assumes that the underlying signal can be described by
u(t) = a + b*t +
c*H(t-t_eq) +
d*H(t-t_eq)*(t-t_eq) +
e_i*log(1 + t/tau_i)
and that the observation contains seasonal signals and jumps at
indicated times. So the observation equation is
uobs(t) = a + b*t +
c*H(t-t_eq) +
d_i*log(1 + t/tau_i) +
f*sin(2*pi*t) + g*sin(4*pi*t)
h*cos(2*pi*t) + m*cos(4*pi*t)
n_i*H(t-t_i)
This function estimates a,b,c,d_i,tau_i,e,f,g,h, and m_i with
a nonlinear least squares algorithm. We put a positivity constraint
on tau and we also restrict c and d_i to have the same sign
'''
# remove jumps that are not within the observation time interval
jumps = np.array([j for j in jumps if (j>t[0]) & (j<t[-1])])
J = len(jumps)
# total number of model parameters
M = 11 + J
def system(m,diff=0):
if diff == 0:
out = np.zeros(len(t))
out += m[0]
out += m[1]*t
out += m[2]*_H(t-start)
out += m[3]*_pslog((t-start)/m[4])
out += m[5]*_psexp((t-start)/m[6])
out += m[7]*np.sin(2*np.pi*t)
out += m[8]*np.sin(4*np.pi*t)
out += m[9]*np.cos(2*np.pi*t)
out += m[10]*np.cos(4*np.pi*t)
for j,val in enumerate(jumps):
out += m[11+j]*_H(t-val)
# derivative w.r.t. t
elif diff == 1:
out = np.zeros(len(t))
out += m[1]
out += _H(t-start)*(m[3]/((t-start) + m[4]))
out += _H(t-start)*(m[5]*np.exp(-(t-start)/m[6])/m[6])
out += m[7]*np.cos(2*np.pi*t)*2*np.pi
out += m[8]*np.cos(4*np.pi*t)*4*np.pi
out += -m[9]*np.sin(2*np.pi*t)*2*np.pi
out += -m[10]*np.sin(4*np.pi*t)*4*np.pi
return out
if detrend:
idx1 = np.array([0,1,7,8,9,10])
idx2 = np.arange(11,M,dtype=int)
nuisance_indices = np.concatenate((idx1,idx2))
signal_indices = np.array([2,3,4,5,6])
else:
idx1 = np.array([7,8,9,10])
idx2 = np.arange(11,M,dtype=int)
nuisance_indices = np.concatenate((idx1,idx2))
signal_indices = np.array([0,1,2,3,4,5,6])
jacobian = modest.make_jacobian(system)
best_err = np.inf
for i in range(trials):
# set lower and upper bounds for model parameters
minit = np.ones(M)
lb = -1e10*np.ones(M)
ub = 1e10*np.ones(M)
# do not allow timescales less that 0.001
minit[4] = 10**np.random.normal(0.0,0.5)
minit[6] = 10**np.random.normal(0.0,0.5)
lb[4] = 1e-3
lb[6] = 1e-3
err1,m1,mcov1,pred1 = modest.nonlin_lstsq(
system,u,minit,
solver=modest.bvls,solver_args=(lb,ub),
data_covariance=var,
LM_damping=True,
regularization=(0,reg),
output=['misfit','solution','solution_covariance','predicted'])
if err1 < best_err:
best_err = err1
m = m1
mcov = mcov1
pred = pred1
# find the jacobian matrix for the final model estimate
jac = jacobian(m,diff=diff)
# find the estimated signal by evaluating the system with 0.0 for
# the nuisance parameters
m[nuisance_indices] = 0.0
signal_pred = system(m,diff=diff)
# slice the model covariance matrix and jacobian so that it only
# contains columns for the parameters of interest
jac = jac[:,signal_indices]
mcov = mcov[np.ix_(signal_indices,signal_indices)]
# use error propagation to find the uncertainty on the predicted
# signal
signal_cov = jac.dot(mcov).dot(jac.T)
# for some reason, diag returns a read-only array
signal_var = np.copy(np.diag(signal_cov))
return signal_pred,signal_var
