def Sy(self, meas, geom):
"""Calculate measurement error covariance.
Input: meas, the instrument measurement
Returns: Sy, the measurement error covariance due to instrument noise
"""
if self.model_type == 'SNR':
bad = meas < 1e-5
meas[bad] = 1e-5
nedl = (1.0 / self.snr) * meas
return pow(s.diagflat(nedl), 2)
elif self.model_type == 'parametric':
nedl = abs(self.noise[:, 0] * s.sqrt(self.noise[:, 1] + meas) +
self.noise[:, 2])
nedl = nedl / s.sqrt(self.integrations)
return pow(s.diagflat(nedl), 2)
elif self.model_type == 'pushbroom':
if geom.pushbroom_column is None:
C = s.squeeze(self.covs.mean(axis=0))
else:
C = self.covs[geom.pushbroom_column, :, :]
return C / s.sqrt(self.integrations)
def dmeas_dinstrumentb(self, x_instrument, wl_hi, rdn_hi):
"""Jacobian of radiance with respect to the instrument parameters
that are unknown and not retrieved, i.e., the inevitable persisting
uncertainties in instrument spectral and radiometric calibration.
Input: meas, a vector of size n_chan
Returns: Kb_instrument, a matrix of size [n_measurements x nb_instrument]
"""
# Uncertainty due to radiometric calibration
meas = self.sample(x_instrument, wl_hi, rdn_hi)
dmeas_dinstrument = s.hstack(
(s.diagflat(meas), s.zeros((self.n_chan, 2))))
# Uncertainty due to spectral calibration
if self.bval[-2] > 1e-6:
dmeas_dinstrument[:, -2] = self.sample(
x_instrument, wl_hi, s.hstack((s.diff(rdn_hi), s.array([0]))))
# Uncertainty due to spectral stray light
if self.bval[-1] > 1e-6:
ssrf = srf(s.arange(-10, 11), 0, 4)
blur = convolve(meas, ssrf, mode='same')
dmeas_dinstrument[:, -1] = blur - meas
return dmeas_dinstrument
def data_dump(self, x, rdn_meas, geom, fname):
"""Dump diagnostic data to a file."""
Seps_inv, Seps_inv_sqrt = self.iv.calc_Seps(rdn_meas, geom)
rdn_est = self.iv.fm.calc_rdn(x, geom)
rdn_est_window = rdn_est[self.winidx]
meas_window = rdn_meas[self.winidx]
meas_resid = (rdn_est_window-meas_window).dot(Seps_inv_sqrt)
xa, Sa, Sa_inv, Sa_inv_sqrt = self.iv.calc_prior(x, geom)
prior_resid = (x - xa).dot(Sa_inv_sqrt)
xopt, coeffs = self.iv.fm.invert_algebraic(x, rdn_meas, geom)
rhoatm, sphalb, transm, solar_irr, coszen = coeffs
Ls = self.iv.fm.surface.calc_Ls(x[self.iv.fm.surface_inds], geom)
# jacobian of cost
Kb = self.iv.fm.Kb(rdn_meas, geom)
K = self.iv.fm.K(x, geom)
xinit = self.iv.fm.init(rdn_meas, geom)
Sy = self.iv.fm.instrument.Sy(rdn_meas, geom)
S_hat, K, G = self.iv.calc_posterior(x, geom, rdn_meas)
lrfl_est = self.iv.fm.calc_lrfl(x, geom)
cost_jac_prior = s.diagflat(x - xa).dot(Sa_inv_sqrt)
cost_jac_meas = Seps_inv_sqrt.dot(K[self.winidx, :])
meas_Cov = self.iv.fm.Seps(rdn_meas, geom)
mdict = {'K': K, 'G': G, 'S_hat': S_hat, 'prior_mu': xa, 'Ls': Ls,
'prior_Cov': Sa, 'rdn_meas': rdn_meas, 'rdn_est': rdn_est,
'x': x, 'meas_Cov': meas_Cov, 'wl': self.iv.fm.wl,
'lrfl_est': lrfl_est, 'cost_jac_prior': cost_jac_prior,
'Kb': Kb, 'cost_jac_meas': cost_jac_meas, 'winidx': self.winidx,
'meas_resid': meas_resid, 'prior_resid': prior_resid,
'noise_Cov': Sy, 'xinit': xinit, 'rhoatm': rhoatm,
'sphalb': sphalb, 'transm': transm, 'solar_irr': solar_irr,
'coszen': coszen}
savemat(fname, mdict)
def Diagonal(diag):
"""
DiagonalMatrix makes a diagonal matrix (clearly) with elements equal to the
elements of diag.
diag: array of elements for diagonal
"""
return scipy.diagflat(diag)
def Diagonal(diag):
"""
DiagonalMatrix makes a diagonal matrix (clearly) with elements equal to the
elements of diag.
diag: array of elements for diagonal
"""
return scipy.diagflat(diag)
def diag(ma):
"""Diagonal matrix with ma on the diagonal.
ma needs to be (1, k) or (k, 1).
"""
EXC._assertVectorOrScalar(asmatrix(ma))
ma = N.mat(ma)
return S.diagflat(ma)
def Kb_instrument(self, meas):
"""Jacobian of radiance with respect to NOT RETRIEVED instrument
variables (relative miscalibration error).
Input: meas, a vector of size nchans
Returns: Kb_instrument, a matrix of size
[n_measurements x nb_instrument]"""
Kb_instrument = s.diagflat(meas)
return Kb_instrument
def __init__(self, config):
# Build the instrument model
self.instrument = Instrument(config['instrument'])
self.n_meas = self.instrument.n_chan
# Build the radiative transfer model
self.RT = RadiativeTransfer(config['radiative_transfer'])
# Build the surface model
self.surface = None
for key, module, cname in surface_models:
module = "isofit." + module
if key in config:
self.surface = getattr(import_module(module),
cname)(config[key])
if self.surface is None:
raise ValueError('Must specify a valid surface model')
# Set up passthrough option
bounds, scale, init, statevec = [], [], [], []
# Build state vector for each part of our forward model
for name in ['surface', 'RT', 'instrument']:
obj = getattr(self, name)
inds = len(statevec) + s.arange(len(obj.statevec), dtype=int)
setattr(self, 'idx_%s' % name, inds)
for b in obj.bounds:
bounds.append(deepcopy(b))
for c in obj.scale:
scale.append(deepcopy(c))
for v in obj.init:
init.append(deepcopy(v))
for v in obj.statevec:
statevec.append(deepcopy(v))
self.bounds = tuple(s.array(bounds).T)
self.scale = s.array(scale)
self.init = s.array(init)
self.statevec = statevec
self.nstate = len(self.statevec)
# Capture unmodeled variables.
bvec, bval = [], []
for name in ['RT', 'instrument', 'surface']:
obj = getattr(self, name)
inds = len(bvec) + s.arange(len(obj.bvec), dtype=int)
setattr(self, '%s_b_inds' % name, inds)
for b in obj.bval:
bval.append(deepcopy(b))
for v in obj.bvec:
bvec.append(deepcopy(v))
self.bvec = s.array(bvec)
self.nbvec = len(self.bvec)
self.bval = s.array(bval)
self.Sb = s.diagflat(pow(self.bval, 2))
def derritz(n,r):
global T, phi
phi=mat(sp.zeros((n,n)))
y=F*r
b=sqrt((y.T*y)[0,0])
y=y/b
phi[:,0]=y[:,0]
for i in range(1,n):
j=i-1
y=F*phi[:,j]
alpha=y.T*phi
for k in range(i):
y=y-alpha[0,k]*phi[:,k]
b=sqrt((y.T*y)[0,0])
phi[:,i]=y[:,0]/b
T=phi.T*F*phi
T=sp.diagflat(sp.diag(T))+sp.diagflat(sp.diag(T,1),1)+sp.diagflat(sp.diag(T,-1),-1)
def adj_loglikelihood(xVec, lenSampleRibo, lenSampleRna, X, y, mu, sign):
disp = sp.hstack([sp.repeat(xVec[0], lenSampleRibo), sp.repeat(xVec[1], lenSampleRna)])
n = 1 / disp
p = n / (n + mu)
loglik = sum(nbinom.logpmf(y, n, p))
diagVec = mu / (1 + sp.dot(mu.transpose(), disp))
diagWM = sp.diagflat(diagVec)
xtwx = sp.dot(sp.dot(sp.transpose(X), diagWM), X)
coxreid = 0.5 * sp.log(sp.linalg.det(xtwx))
ret = (loglik - coxreid) * sign
#print "return value is " + str(ret)
if isinstance(ret, complex):
raise complexException()
return ret
def adj_loglikelihood(xVec, lenSampleRibo, lenSampleRna, X, y, mu, sign):
disp = sp.hstack(
[sp.repeat(xVec[0], lenSampleRibo),
sp.repeat(xVec[1], lenSampleRna)])
n = 1 / disp
p = n / (n + mu)
loglik = sum(nbinom.logpmf(y, n, p))
diagVec = mu / (1 + sp.dot(mu.transpose(), disp))
diagWM = sp.diagflat(diagVec)
xtwx = sp.dot(sp.dot(sp.transpose(X), diagWM), X)
coxreid = 0.5 * sp.log(sp.linalg.det(xtwx))
ret = (loglik - coxreid) * sign
#print "return value is " + str(ret)
if isinstance(ret, complex):
raise complexException()
return ret
def column_covariances(X, uniformity_thresh):
Xvert = high_frequency_vert(X, sigma=4.0)
Xvertp = high_frequency_vert(X, sigma=3.0)
models = []
use_C = []
for i in range(X.shape[2]):
xsub = Xvert[:, :, i]
xsubp = Xvertp[:, :, i]
mu = xsub.mean(axis=0)
dists = s.sqrt(pow((xsub - mu), 2).sum(axis=1))
distsp = s.sqrt(pow((xsubp - mu), 2).sum(axis=1))
thresh = percentile(dists, 95.0)
uthresh = dists * uniformity_thresh
#use = s.logical_and(dists<thresh, abs(dists-distsp) < uthresh)
use = dists < thresh
C = s.cov(xsub[use, :], rowvar=False)
[U, V, D] = svd(C)
V[V < 1e-8] = 1e-8
C = U.dot(s.diagflat(V)).dot(D)
models.append(C)
use_C.append(use)
return s.array(models), Xvert, Xvertp, s.array(use_C).T
def __init__(self, config):
'''ForwardModel contains all the information about how to calculate
radiance measurements at a specific spectral calibration, given a
state vector. It also manages the distributions of unretrieved,
unknown parameters of the state vector (i.e. the S_b and K_b
matrices of Rodgers et al.'''
self.instrument = Instrument(config['instrument'])
self.n_meas = self.instrument.n_chan
# Build the radiative transfer model
self.RT = None
for key, module, cname in RT_models:
if key in config:
self.RT = getattr(import_module(module), cname)(config[key])
if self.RT is None:
raise ValueError('Must specify a valid radiative transfer model')
# Build the surface model
self.surface = None
for key, module, cname in surface_models:
if key in config:
self.surface = getattr(import_module(module),
cname)(config[key])
if self.surface is None:
raise ValueError('Must specify a valid surface model')
# Set up passthrough option
bounds, scale, init, statevec = [], [], [], []
# Build state vector for each part of our forward model
for name in ['surface', 'RT', 'instrument']:
obj = getattr(self, name)
inds = len(statevec) + s.arange(len(obj.statevec), dtype=int)
setattr(self, 'idx_%s' % name, inds)
for b in obj.bounds:
bounds.append(deepcopy(b))
for c in obj.scale:
scale.append(deepcopy(c))
for v in obj.init:
init.append(deepcopy(v))
for v in obj.statevec:
statevec.append(deepcopy(v))
self.bounds = tuple(s.array(bounds).T)
self.scale = s.array(scale)
self.init = s.array(init)
self.statevec = statevec
self.nstate = len(self.statevec)
# Capture unmodeled variables
bvec, bval = [], []
for name in ['RT', 'instrument', 'surface']:
obj = getattr(self, name)
inds = len(bvec) + s.arange(len(obj.bvec), dtype=int)
setattr(self, '%s_b_inds' % name, inds)
for b in obj.bval:
bval.append(deepcopy(b))
for v in obj.bvec:
bvec.append(deepcopy(v))
self.bvec = s.array(bvec)
self.nbvec = len(self.bvec)
self.bval = s.array(bval)
self.Sb = s.diagflat(pow(self.bval, 2))
return
def Sa(self):
"""Covariance of prior distribution (diagonal)."""
if self.n_state == 0:
return s.zeros((0, 0), dtype=float)
return s.diagflat(pow(self.prior_sigma, 2))
def __init__(self, config):
'''ForwardModel contains all the information about how to calculate
radiance measurements at a specific spectral calibration, given a
state vector. It also manages the distributions of unretrieved,
unknown parameters of the state vector (i.e. the S_b and K_b
matrices of Rodgers et al.'''
self.instrument = Instrument(config['instrument'])
# Build the radiative transfer model
if 'modtran_radiative_transfer' in config:
self.RT = ModtranRT(config['modtran_radiative_transfer'],
self.instrument)
elif 'libradtran_radiative_transfer' in config:
self.RT = LibRadTranRT(config['libradtran_radiative_transfer'],
self.instrument)
else:
raise ValueError('Must specify a valid radiative transfer model')
# Build the surface model
if 'surface' in config:
self.surface = Surface(config['surface'], self.RT)
elif 'multicomponent_surface' in config:
self.surface = MultiComponentSurface(
config['multicomponent_surface'], self.RT)
elif 'emissive_surface' in config:
self.surface = MixBBSurface(config['emissive_surface'], self.RT)
elif 'glint_surface' in config:
self.surface = GlintSurface(config['glint_surface'], self.RT)
elif 'iop_surface' in config:
self.surface = IOPSurface(config['iop_surface'], self.RT)
else:
raise ValueError('Must specify a valid surface model')
bounds, scale, init_val, statevec = [], [], [], []
# Build state vector for each part of our forward model
for name in ['surface', 'RT']:
obj = getattr(self, name)
inds = len(statevec) + s.arange(len(obj.statevec), dtype=int)
setattr(self, '%s_inds' % name, inds)
for b in obj.bounds:
bounds.append(deepcopy(b))
for c in obj.scale:
scale.append(deepcopy(c))
for v in obj.init_val:
init_val.append(deepcopy(v))
for v in obj.statevec:
statevec.append(deepcopy(v))
self.bounds = tuple(s.array(bounds).T)
self.scale = s.array(scale)
self.init_val = s.array(init_val)
self.statevec = statevec
self.wl = self.instrument.wl
# Capture unmodeled variables
bvec, bval = [], []
for name in ['RT', 'instrument', 'surface']:
obj = getattr(self, name)
inds = len(bvec) + s.arange(len(obj.bvec), dtype=int)
setattr(self, '%s_b_inds' % name, inds)
for b in obj.bval:
bval.append(deepcopy(b))
for v in obj.bvec:
bvec.append(deepcopy(v))
self.bvec = s.array(bvec)
self.bval = s.array(bval)
self.Sb = s.diagflat(pow(self.bval, 2))
return
#%% Compute the EKF results
from KalmanFilterClass import LinearKalmanFilter2D, Data
batch_kalman = []
for i in range(batch_y.shape[0]):
deltaT = sp.mean(t[1:] - t[0:-1])
state0 = sp.squeeze(batch_x[i,0,:])
P0 = sp.identity(4)*0.1
F0 = sp.array([[1, 0, deltaT, 0],\
[0, 1, 0, deltaT],\
[0, 0, 1, 0],\
[0, 0, 0, 1]])
H0 = sp.identity(4)
Q0 = sp.diagflat([0.0005,0.0005,0.1,0.1])
<<<<<<< HEAD
R0 = sp.diagflat([0.07,0.07,0.07,0.07])
=======
R0 = sp.diagflat([7,7,7,7])
>>>>>>> ce7f7d394a400190409def1c7004335ae3bc04a8
data = Data(sp.squeeze(batch_x[i,:,0]),sp.squeeze(batch_x[i,:,1]),sp.squeeze(batch_x[i,:,2]),sp.squeeze(batch_x[i,:,3]),[],[])
filter1b = LinearKalmanFilter2D(F0, H0, P0, Q0, R0, state0)
kalman_data = filter1b.process_data(data)
batch_kalman.append(sp.vstack([kalman_data.x[1:],kalman_data.y[1:], kalman_data.vx[1:], kalman_data.vy[1:]]).T)
xk_batch = sp.stack(batch_kalman)
xk_batch - batch_y
print('Kalman loss;'.ljust(12), sp.mean(pow(xk_batch[BATCH_SIZE:,:,:] - batch_y[BATCH_SIZE:,:,:],2)))
#%% Compute the EKF results
from KalmanFilterClass import LinearKalmanFilter3D, Data3D
batch_kalman = []
for i in range(batch_y.shape[0]):
deltaT = sp.mean(t[1:] - t[0:-1])
state0 = sp.squeeze(batch_x[i, 0, :])
P0 = sp.identity(6) * 0.0001
F0 = sp.array([[1, 0, 0, deltaT, 0, 0],\
[0, 1, 0, 0, deltaT, 0],\
[0, 0, 1, 0, 0, deltaT],\
[0, 0, 0, 1, 0, 0],\
[0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 1]])
H0 = sp.identity(6)
Q0 = sp.diagflat([0.0001, 0.0001, 0.0001, 0.1, 0.1, 0.1])
R0 = sp.diagflat([6.0, 6.0, 6.0, 0.5, 0.5, 0.5])
filter3d = LinearKalmanFilter3D(F0, H0, P0, Q0, R0, state0)
data = Data3D(sp.squeeze(batch_x[i, :, 0]), sp.squeeze(batch_x[i, :, 1]),
sp.squeeze(batch_x[i, :, 2]), sp.squeeze(batch_x[i, :, 3]),
[], [])
filter3d = LinearKalmanFilter3D(F0, H0, P0, Q0, R0, state0)
kalman_data = filter3d.process_data(data)
batch_kalman.append(
sp.vstack([
kalman_data.x[1:], kalman_data.y[1:], kalman_data.z[1:],
kalman_data.vx[1:], kalman_data.vy[1:], kalman_data.vz[1:]
]).T)
xk_batch = sp.stack(batch_kalman)
def Sa(self):
"""Pull the priors from each of the individual RTs.
"""
return s.diagflat(pow(s.array(self.prior_sigma), 2))
M=mat(sp.eye(n))
K=mat(sp.eye(n))*2
K[n-1,n-1]=1.
for i in range(n-1):
K[i+1,i]=-1.
K[i,i+1]=-1.
F=K.I
lp(M,'mat',title="Mass matrix (=I) M")
lp(K,'mat',title="Stiffness matrix K")
lp(F,'mat',title="Flex.lity matrix F")
# evecs are normalized with respect to M
evals, evecs = eigh(K,M)
L=mat(sp.diagflat(evals))
lp(L,'mat',title="Eigenvalues Matrix L")
lp(evecs,'mat',title="Eigenvectors matrix, \Psi")
for r in ( sp.mat((0,0,0,0.,1.)).T,sp.mat((0,0,0,-2.,1.)).T, sp.mat((1,1,1,1.,1.)).T):
lp(r.T,'vet',title="Load vector transposed r^T")
derritz(n,r)
lp(phi,'mat',title="Derived Ritz Vectors matrix \Phi")
# lp(T,'mat',title="Tridiagonal DRV matrix, T")
Gamma=evecs.T*r
# lp(Gamma.T,'vet',title="Modal partecipation factors")
Gammh=phi.T*r
# lp(Gammh.T,'vet',title="DRV's partecipation factors")
def Sa(self):
'''Covariance of prior distribution. Our state vector covariance
is diagonal with very loose constraints.'''
if self.n_state == 0:
return s.zeros((0, 0), dtype=float)
return s.diagflat(pow(self.prior_sigma, 2))
def Sa(self):
'''Covariance of prior distribution. Our state vector covariance is
diagonal with very loose constraints.'''
std_factor = 10.0
return s.diagflat(pow(s.diff(self.bounds) * std_factor, 2))
def flatDiags(n):
x = sp.zeros((n,n))
for i in range(n):
x += sp.diagflat(sp.ones(n-i)*(i+1),i)
return x
def diagFlatThree(n):
return sp.diagflat(sp.arange(1,n+1,1))
def totalDiags(n):
x = sp.zeros((n,n))
for i in range(n):
x+= sp.diagflat(sp.ones(n-i)/float((i+1)),i) + sp.diagflat(sp.ones(n-i)/float((i+1)),-i)
x -= sp.diag(sp.ones(n),0)
return x
def write_spectrum(self,
row,
col,
states,
meas,
geom,
flush_immediately=False):
"""Write data from a single inversion to all output buffers."""
self.writes = self.writes + 1
if len(states) == 0:
# Write a bad data flag
atm_bad = s.zeros(len(self.fm.statevec)) * -9999.0
state_bad = s.zeros(len(self.fm.statevec)) * -9999.0
data_bad = s.zeros(self.fm.instrument.n_chan) * -9999.0
to_write = {
'estimated_state_file': state_bad,
'estimated_reflectance_file': data_bad,
'estimated_emission_file': data_bad,
'modeled_radiance_file': data_bad,
'apparent_reflectance_file': data_bad,
'path_radiance_file': data_bad,
'simulated_measurement_file': data_bad,
'algebraic_inverse_file': data_bad,
'atmospheric_coefficients_file': atm_bad,
'radiometry_correction_file': data_bad,
'spectral_calibration_file': data_bad,
'posterior_uncertainty_file': state_bad
}
else:
# The inversion returns a list of states, which are
# intepreted either as samples from the posterior (MCMC case)
# or as a gradient descent trajectory (standard case). For
# gradient descent the last spectrum is the converged solution.
if self.iv.method == 'MCMC':
state_est = states.mean(axis=0)
else:
state_est = states[-1, :]
# Spectral calibration
wl, fwhm = self.fm.calibration(state_est)
cal = s.column_stack(
[s.arange(0, len(wl)), wl / 1000.0, fwhm / 1000.0])
# If there is no actual measurement, we use the simulated version
# in subsequent calculations. Naturally in these cases we're
# mostly just interested in the simulation result.
if meas is None:
meas = self.fm.calc_rdn(state_est, geom)
# Rodgers diagnostics
lamb_est, meas_est, path_est, S_hat, K, G = \
self.iv.forward_uncertainty(state_est, meas, geom)
# Simulation with noise
meas_sim = self.fm.instrument.simulate_measurement(meas_est, geom)
# Algebraic inverse and atmospheric optical coefficients
x_surface, x_RT, x_instrument = self.fm.unpack(state_est)
rfl_alg_opt, Ls, coeffs = invert_algebraic(
self.fm.surface, self.fm.RT, self.fm.instrument, x_surface,
x_RT, x_instrument, meas, geom)
rhoatm, sphalb, transm, solar_irr, coszen, transup = coeffs
L_atm = self.fm.RT.get_L_atm(x_RT, geom)
L_down_transmitted = self.fm.RT.get_L_down_transmitted(x_RT, geom)
L_up = self.fm.RT.get_L_up(x_RT, geom)
atm = s.column_stack(
list(coeffs[:4]) + [s.ones((len(wl), 1)) * coszen])
# Upward emission & glint and apparent reflectance
Ls_est = self.fm.calc_Ls(state_est, geom)
apparent_rfl_est = lamb_est + Ls_est
# Radiometric calibration
factors = s.ones(len(wl))
if 'radiometry_correction_file' in self.outfiles:
if 'reference_reflectance_file' in self.infiles:
reference_file = self.infiles['reference_reflectance_file']
self.rfl_ref = reference_file.read_spectrum(row, col)
self.wl_ref = reference_file.wl
w, fw = self.fm.instrument.calibration(x_instrument)
resamp = resample_spectrum(self.rfl_ref,
self.wl_ref,
w,
fw,
fill=True)
meas_est = self.fm.calc_meas(state_est, geom, rfl=resamp)
factors = meas_est / meas
else:
logging.warning('No reflectance reference')
# Assemble all output products
to_write = {
'estimated_state_file':
state_est,
'estimated_reflectance_file':
s.column_stack((self.fm.surface.wl, lamb_est)),
'estimated_emission_file':
s.column_stack((self.fm.surface.wl, Ls_est)),
'estimated_reflectance_file':
s.column_stack((self.fm.surface.wl, lamb_est)),
'modeled_radiance_file':
s.column_stack((wl, meas_est)),
'apparent_reflectance_file':
s.column_stack((self.fm.surface.wl, apparent_rfl_est)),
'path_radiance_file':
s.column_stack((wl, path_est)),
'simulated_measurement_file':
s.column_stack((wl, meas_sim)),
'algebraic_inverse_file':
s.column_stack((self.fm.surface.wl, rfl_alg_opt)),
'atmospheric_coefficients_file':
atm,
'radiometry_correction_file':
factors,
'spectral_calibration_file':
cal,
'posterior_uncertainty_file':
s.sqrt(s.diag(S_hat))
}
for product in self.outfiles:
logging.debug('IO: Writing ' + product)
self.outfiles[product].write_spectrum(row, col, to_write[product])
if (self.writes % flush_rate) == 0 or flush_immediately:
self.outfiles[product].flush_buffers()
# Special case! samples file is matlab format.
if 'mcmc_samples_file' in self.output:
logging.debug('IO: Writing mcmc_samples_file')
mdict = {'samples': states}
s.io.savemat(self.output['mcmc_samples_file'], mdict)
# Special case! Data dump file is matlab format.
if 'data_dump_file' in self.output:
logging.debug('IO: Writing data_dump_file')
x = state_est
xall = states
Seps_inv, Seps_inv_sqrt = self.iv.calc_Seps(x, meas, geom)
meas_est_window = meas_est[self.iv.winidx]
meas_window = meas[self.iv.winidx]
xa, Sa, Sa_inv, Sa_inv_sqrt = self.iv.calc_prior(x, geom)
prior_resid = (x - xa).dot(Sa_inv_sqrt)
rdn_est = self.fm.calc_rdn(x, geom)
rdn_est_all = s.array(
[self.fm.calc_rdn(xtemp, geom) for xtemp in states])
x_surface, x_RT, x_instrument = self.fm.unpack(x)
Kb = self.fm.Kb(x, geom)
xinit = invert_simple(self.fm, meas, geom)
Sy = self.fm.instrument.Sy(meas, geom)
cost_jac_prior = s.diagflat(x - xa).dot(Sa_inv_sqrt)
cost_jac_meas = Seps_inv_sqrt.dot(K[self.iv.winidx, :])
meas_Cov = self.fm.Seps(x, meas, geom)
lamb_est, meas_est, path_est, S_hat, K, G = \
self.iv.forward_uncertainty(state_est, meas, geom)
A = s.matmul(K, G)
# Form the MATLAB dictionary object and write to file
mdict = {
'K': K,
'G': G,
'S_hat': S_hat,
'prior_mu': xa,
'Ls': Ls,
'prior_Cov': Sa,
'meas': meas,
'rdn_est': rdn_est,
'rdn_est_all': rdn_est_all,
'x': x,
'xall': xall,
'x_surface': x_surface,
'x_RT': x_RT,
'x_instrument': x_instrument,
'meas_Cov': meas_Cov,
'wl': wl,
'fwhm': fwhm,
'lamb_est': lamb_est,
'coszen': coszen,
'cost_jac_prior': cost_jac_prior,
'Kb': Kb,
'A': A,
'cost_jac_meas': cost_jac_meas,
'winidx': self.iv.winidx,
'windows': self.iv.windows,
'prior_resid': prior_resid,
'noise_Cov': Sy,
'xinit': xinit,
'rhoatm': rhoatm,
'sphalb': sphalb,
'transm': transm,
'transup': transup,
'solar_irr': solar_irr,
'L_atm': L_atm,
'L_down_transmitted': L_down_transmitted,
'L_up': L_up
}
s.io.savemat(self.output['data_dump_file'], mdict)
# Write plots, if needed
if len(states) > 0 and 'plot_directory' in self.output:
if 'reference_reflectance_file' in self.infiles:
reference_file = self.infiles['reference_reflectance_file']
self.rfl_ref = reference_file.read_spectrum(row, col)
self.wl_ref = reference_file.wl
for i, x in enumerate(states):
# Calculate intermediate solutions
lamb_est, meas_est, path_est, S_hat, K, G = \
self.iv.forward_uncertainty(state_est, meas, geom)
plt.cla()
red = [0.7, 0.2, 0.2]
wl, fwhm = self.fm.calibration(x)
xmin, xmax = min(wl), max(wl)
fig = plt.subplots(1, 2, figsize=(10, 5))
plt.subplot(1, 2, 1)
meas_est = self.fm.calc_meas(x, geom)
for lo, hi in self.iv.windows:
idx = s.where(s.logical_and(wl > lo, wl < hi))[0]
p1 = plt.plot(wl[idx], meas[idx], color=red, linewidth=2)
p2 = plt.plot(wl, meas_est, color='k', linewidth=1)
plt.title("Radiance")
plt.title("Measurement (Scaled DN)")
ymax = max(meas) * 1.25
ymax = max(meas) + 0.01
ymin = min(meas) - 0.01
plt.text(500, ymax * 0.92, "Measured", color=red)
plt.text(500, ymax * 0.86, "Model", color='k')
plt.ylabel(r"$\mu$W nm$^{-1}$ sr$^{-1}$ cm$^{-2}$")
plt.ylabel("Intensity")
plt.xlabel("Wavelength (nm)")
plt.ylim([-0.001, ymax])
plt.ylim([ymin, ymax])
plt.xlim([xmin, xmax])
plt.subplot(1, 2, 2)
lamb_est = self.fm.calc_lamb(x, geom)
ymax = min(max(lamb_est) * 1.25, 0.10)
for lo, hi in self.iv.windows:
# black line
idx = s.where(s.logical_and(wl > lo, wl < hi))[0]
p2 = plt.plot(wl[idx], lamb_est[idx], 'k', linewidth=2)
ymax = max(max(lamb_est[idx] * 1.2), ymax)
# red line
if 'reference_reflectance_file' in self.infiles:
idx = s.where(
s.logical_and(self.wl_ref > lo,
self.wl_ref < hi))[0]
p1 = plt.plot(self.wl_ref[idx],
self.rfl_ref[idx],
color=red,
linewidth=2)
ymax = max(max(self.rfl_ref[idx] * 1.2), ymax)
# green and blue lines - surface components
if hasattr(self.fm.surface, 'components') and \
self.output['plot_surface_components']:
idx = s.where(
s.logical_and(self.fm.surface.wl > lo,
self.fm.surface.wl < hi))[0]
p3 = plt.plot(self.fm.surface.wl[idx],
self.fm.xa(x, geom)[idx],
'b',
linewidth=2)
for j in range(len(self.fm.surface.components)):
z = self.fm.surface.norm(
lamb_est[self.fm.surface.idx_ref])
mu = self.fm.surface.components[j][0] * z
plt.plot(self.fm.surface.wl[idx],
mu[idx],
'g:',
linewidth=1)
plt.text(500, ymax * 0.86, "Remote estimate", color='k')
if 'reference_reflectance_file' in self.infiles:
plt.text(500, ymax * 0.92, "In situ reference", color=red)
if hasattr(self.fm.surface, 'components') and \
self.output['plot_surface_components']:
plt.text(500, ymax * 0.80, "Prior mean state ", color='b')
plt.text(500,
ymax * 0.74,
"Surface components ",
color='g')
plt.ylim([-0.0010, ymax])
plt.xlim([xmin, xmax])
plt.title("Reflectance")
plt.title("Source Model")
plt.xlabel("Wavelength (nm)")
fn = self.output['plot_directory'] + ('/frame_%i.png' % i)
plt.savefig(fn)
plt.close()
def Sa(self):
'''Covariance of prior distribution. Our state vector covariance
is diagonal with very loose constraints.'''
return s.diagflat(pow(self.prior_sigma, 2))
plt.grid(which='both')
plt.legend()
plt.savefig('training_progress1D.png',bbox_inches='tight', dpi=200)
#%% Compute the EKF results
from KalmanFilterClass import LinearKalmanFilter1D, Data1D
batch_kalman = []
deltaT = sp.mean(t[1:] - t[0:-1])
P0 = sp.identity(2)*0.01
F0 = sp.array([[1, deltaT],\
[0, 1]])
H0 = sp.identity(2)
Q0 = sp.diagflat([0.0001,0.00001])
R0 = sp.diagflat([1.5,0.01])
for i in range(batch_y.shape[0]):
data = Data1D(sp.squeeze(batch_x[i,:,0]),sp.squeeze(batch_x[i,:,1]),[])
state0 = sp.array([0, batch_x[i,0,1]]).T
filter1b = LinearKalmanFilter1D(F0, H0, P0, Q0, R0, state0)
kalman_data = filter1b.process_data(data)
batch_kalman.append(sp.vstack([kalman_data.x[1:], kalman_data.vx[1:]]).T)
xk_batch = sp.stack(batch_kalman)
print(xk_batch.shape)
print('Kalman loss;'.ljust(12), sp.mean(pow(xk_batch[BATCH_SIZE:,:,:] - batch_y[BATCH_SIZE:,:,:],2)))
print(xk_batch.shape)
#%% Plot the fit
plt.figure(figsize=(14,16))
print("For iter %i, learning rate %3.6f" % (itr, learning_rate))
print("Loss ", los)
print("__________________")
itr = itr + 1
#%% Compute the EKF results
from KalmanFilterClass import LinearKalmanFilter1D, Data1D
batch_kalman = []
deltaT = sp.mean(t[1:] - t[0:-1])
state0 = sp.array([0, 0]).T
P0 = sp.identity(2) * 0.1
F0 = sp.array([[1, deltaT],\
[0, 1]])
H0 = sp.identity(2)
Q0 = sp.diagflat([0.005, 0.0001])
R0 = sp.diagflat([0.25, 0.001])
for i in range(BATCH_SIZE):
data = Data1D(sp.squeeze(batch_x[i, :, 0]), sp.squeeze(batch_x[i, :, 1]),
[])
filter1b = LinearKalmanFilter1D(F0, H0, P0, Q0, R0, state0)
kalman_data = filter1b.process_data(data)
batch_kalman.append(sp.vstack([kalman_data.x[1:], kalman_data.vx[1:]]).T)
xk_batch = sp.stack(batch_kalman)
print(xk_batch.shape)
#%% Plot the fit
plt.figure(figsize=(14, 16))
for batch_idx in range(BATCH_SIZE):
out_x = sp.squeeze(out[batch_idx, :, 0])