def load_rt(self, point, fn):
"""Load the results of a LibRadTran run """
wl, rdn0, irr = s.loadtxt(self.lut_dir + '/LUT_' + fn + '_alb0.out').T
wl, rdn025, irr = s.loadtxt(self.lut_dir + '/LUT_' + fn +
'_alb025.out').T
wl, rdn05, irr = s.loadtxt(self.lut_dir + '/LUT_' + fn +
'_alb05.out').T
# Replace a few zeros in the irradiance spectrum via interpolation
good = irr > 1e-15
bad = s.logical_not(good)
irr[bad] = interp1d(wl[good], irr[good])(wl[bad])
# Translate to Top of Atmosphere (TOA) reflectance
rhoatm = rdn0 / 10.0 / irr * s.pi # Translate to uW nm-1 cm-2 sr-1
rho025 = rdn025 / 10.0 / irr * s.pi
rho05 = rdn05 / 10.0 / irr * s.pi
# Resample TOA reflectances to simulate the instrument observation
rhoatm = resample_spectrum(rhoatm, wl, self.wl, self.fwhm)
rho025 = resample_spectrum(rho025, wl, self.wl, self.fwhm)
rho05 = resample_spectrum(rho05, wl, self.wl, self.fwhm)
irr = resample_spectrum(irr, wl, self.wl, self.fwhm)
# Calculate some atmospheric optical constants
sphalb = 2.8 * (2.0 * rho025 - rhoatm - rho05) / (rho025 - rho05)
transm = (rho05 - rhoatm) * (2.0 - sphalb)
# For now, don't estimate this term!!
# TODO: Have LibRadTran calculate it directly
transup = s.zeros(self.wl.shape)
# Get solar zenith, translate to irradiance at zenith = 0
with open(self.lut_dir + '/LUT_' + fn + '.zen', 'r') as fin:
output = fin.read().split()
solzen, solaz = [float(q) for q in output[1:]]
irr = irr / s.cos(solzen / 360.0 * 2.0 * s.pi)
return self.wl, irr, solzen, rhoatm, transm, sphalb, transup
def sample(self, x_instrument, wl_hi, rdn_hi):
""" Apply instrument sampling to a radiance spectrum, returning the
predicted measurement"""
if self.calibration_fixed and all((self.wl_init - wl_hi) < wl_tol):
return rdn_hi
wl, fwhm = self.calibration(x_instrument)
if rdn_hi.ndim == 1:
return resample_spectrum(rdn_hi, wl_hi, wl, fwhm)
else:
resamp = []
# The "fast resample" option approximates a complete resampling
# by a convolution with a uniform FWHM.
if self.fast_resample:
for i, r in enumerate(rdn_hi):
ssrf = srf(s.arange(-10, 11), 0, fwhm[0])
blur = convolve(r, ssrf, mode='same')
resamp.append(interp1d(wl_hi, blur)(wl))
else:
for i, r in enumerate(rdn_hi):
r2 = resample_spectrum(r, wl_hi, wl, fwhm)
resamp.append(r2)
return s.array(resamp)
def load_rt(self, point, fn):
"""Load the results of a SixS run """
with open(os.path.join(self.lut_dir, fn), 'r') as l:
lines = l.readlines()
with open(os.path.join(self.lut_dir, 'LUT_' + fn + '.inp'), 'r') as l:
inlines = l.readlines()
solzen = float(inlines[1].strip().split()[0])
# Strip header
for i, ln in enumerate(lines):
if ln.startswith('* trans down up'):
lines = lines[(i + 1):(i + 1 + self.sixs_ngrid_init)]
break
solzens = s.zeros(len(lines))
sphalbs = s.zeros(len(lines))
transups = s.zeros(len(lines))
transms = s.zeros(len(lines))
rhoatms = s.zeros(len(lines))
self.grid = s.zeros(len(lines))
for i, ln in enumerate(lines):
ln = ln.replace('*', ' ').strip()
w, gt, scad, scau, salb, rhoa, swl, step, sbor, dsol, toar = \
ln.split()
self.grid[i] = float(w) * 1000.0 # convert to nm
solzens[i] = float(solzen)
sphalbs[i] = float(salb)
transups[i] = 0.0 # float(scau)
transms[i] = float(scau) * float(scad) * float(gt)
rhoatms[i] = float(rhoa)
solzen = resample_spectrum(solzens, self.grid, self.wl, self.fwhm)
rhoatm = resample_spectrum(rhoatms, self.grid, self.wl, self.fwhm)
transm = resample_spectrum(transms, self.grid, self.wl, self.fwhm)
sphalb = resample_spectrum(sphalbs, self.grid, self.wl, self.fwhm)
transup = resample_spectrum(transups, self.grid, self.wl, self.fwhm)
irr = resample_spectrum(self.irr, self.iwl, self.wl, self.fwhm)
return self.wl, irr, solzen, rhoatm, transm, sphalb, transup
def write_spectrum(self, row, col, states, meas, geom):
"""Write data from a single inversion to all output buffers."""
self.writes = self.writes + 1
if len(states) == 0:
# Write a bad data flag
atm_bad = s.zeros(len(self.fm.statevec)) * -9999.0
state_bad = s.zeros(len(self.fm.statevec)) * -9999.0
data_bad = s.zeros(self.fm.instrument.n_chan) * -9999.0
to_write = {
'estimated_state_file': state_bad,
'estimated_reflectance_file': data_bad,
'estimated_emission_file': data_bad,
'modeled_radiance_file': data_bad,
'apparent_reflectance_file': data_bad,
'path_radiance_file': data_bad,
'simulated_measurement_file': data_bad,
'algebraic_inverse_file': data_bad,
'atmospheric_coefficients_file': atm_bad,
'radiometry_correction_file': data_bad,
'spectral_calibration_file': data_bad,
'posterior_uncertainty_file': state_bad
}
else:
# The inversion returns a list of states, which are
# intepreted either as samples from the posterior (MCMC case)
# or as a gradient descent trajectory (standard case). For
# gradient descent the last spectrum is the converged solution.
if self.iv.method == 'MCMC':
state_est = states.mean(axis=0)
else:
state_est = states[-1, :]
# Spectral calibration
wl, fwhm = self.fm.calibration(state_est)
cal = s.column_stack(
[s.arange(0, len(wl)), wl / 1000.0, fwhm / 1000.0])
# If there is no actual measurement, we use the simulated version
# in subsequent calculations. Naturally in these cases we're
# mostly just interested in the simulation result.
if meas is None:
meas = self.fm.calc_rdn(state_est, geom)
# Rodgers diagnostics
lamb_est, meas_est, path_est, S_hat, K, G = \
self.iv.forward_uncertainty(state_est, meas, geom)
# Simulation with noise
meas_sim = self.fm.instrument.simulate_measurement(meas_est, geom)
# Algebraic inverse and atmospheric optical coefficients
x_surface, x_RT, x_instrument = self.fm.unpack(state_est)
rfl_alg_opt, Ls, coeffs = invert_algebraic(
self.fm.surface, self.fm.RT, self.fm.instrument, x_surface,
x_RT, x_instrument, meas, geom)
rhoatm, sphalb, transm, solar_irr, coszen, transup = coeffs
atm = s.column_stack(
list(coeffs[:4]) + [s.ones((len(wl), 1)) * coszen])
# Upward emission & glint and apparent reflectance
Ls_est = self.fm.calc_Ls(state_est, geom)
apparent_rfl_est = lamb_est + Ls_est
# radiometric calibration
factors = s.ones(len(wl))
if 'radiometry_correction_file' in self.outfiles:
if 'reference_reflectance_file' in self.infiles:
reference_file = self.infiles['reference_reflectance_file']
self.rfl_ref = reference_file.read_spectrum(row, col)
self.wl_ref = reference_file.wl
w, fw = self.fm.instrument.calibration(x_instrument)
resamp = resample_spectrum(self.rfl_ref,
self.wl_ref,
w,
fw,
fill=True)
meas_est = self.fm.calc_meas(state_est, geom, rfl=resamp)
factors = meas_est / meas
else:
logging.warning('No reflectance reference')
# Assemble all output products
to_write = {
'estimated_state_file':
state_est,
'estimated_reflectance_file':
s.column_stack((self.fm.surface.wl, lamb_est)),
'estimated_emission_file':
s.column_stack((self.fm.surface.wl, Ls_est)),
'estimated_reflectance_file':
s.column_stack((self.fm.surface.wl, lamb_est)),
'modeled_radiance_file':
s.column_stack((wl, meas_est)),
'apparent_reflectance_file':
s.column_stack((self.fm.surface.wl, apparent_rfl_est)),
'path_radiance_file':
s.column_stack((wl, path_est)),
'simulated_measurement_file':
s.column_stack((wl, meas_sim)),
'algebraic_inverse_file':
s.column_stack((self.fm.surface.wl, rfl_alg_opt)),
'atmospheric_coefficients_file':
atm,
'radiometry_correction_file':
factors,
'spectral_calibration_file':
cal,
'posterior_uncertainty_file':
s.sqrt(s.diag(S_hat))
}
for product in self.outfiles:
logging.debug('IO: Writing ' + product)
self.outfiles[product].write_spectrum(row, col, to_write[product])
if (self.writes % flush_rate) == 0:
self.outfiles[product].flush_buffers()
# Special case! samples file is matlab format.
if 'mcmc_samples_file' in self.output:
logging.debug('IO: Writing mcmc_samples_file')
mdict = {'samples': states}
s.io.savemat(self.output['mcmc_samples_file'], mdict)
# Special case! Data dump file is matlab format.
if 'data_dump_file' in self.output:
logging.debug('IO: Writing data_dump_file')
x = state_est
Seps_inv, Seps_inv_sqrt = self.iv.calc_Seps(x, meas, geom)
meas_est_window = meas_est[self.iv.winidx]
meas_window = meas[self.iv.winidx]
xa, Sa, Sa_inv, Sa_inv_sqrt = self.iv.calc_prior(x, geom)
prior_resid = (x - xa).dot(Sa_inv_sqrt)
rdn_est = self.fm.calc_rdn(x, geom)
x_surface, x_RT, x_instrument = self.fm.unpack(x)
Kb = self.fm.Kb(x, geom)
xinit = invert_simple(self.fm, meas, geom)
Sy = self.fm.instrument.Sy(meas, geom)
cost_jac_prior = s.diagflat(x - xa).dot(Sa_inv_sqrt)
cost_jac_meas = Seps_inv_sqrt.dot(K[self.iv.winidx, :])
meas_Cov = self.fm.Seps(x, meas, geom)
lamb_est, meas_est, path_est, S_hat, K, G = \
self.iv.forward_uncertainty(state_est, meas, geom)
A = s.matmul(K, G)
# Form the MATLAB dictionary object and write to file
mdict = {
'K': K,
'G': G,
'S_hat': S_hat,
'prior_mu': xa,
'Ls': Ls,
'prior_Cov': Sa,
'meas': meas,
'rdn_est': rdn_est,
'x': x,
'x_surface': x_surface,
'x_RT': x_RT,
'x_instrument': x_instrument,
'meas_Cov': meas_Cov,
'wl': wl,
'fwhm': fwhm,
'lamb_est': lamb_est,
'coszen': coszen,
'cost_jac_prior': cost_jac_prior,
'Kb': Kb,
'A': A,
'cost_jac_meas': cost_jac_meas,
'winidx': self.iv.winidx,
'prior_resid': prior_resid,
'noise_Cov': Sy,
'xinit': xinit,
'rhoatm': rhoatm,
'sphalb': sphalb,
'transm': transm,
'solar_irr': solar_irr
}
s.io.savemat(self.output['data_dump_file'], mdict)
# Write plots, if needed
if len(states) > 0 and 'plot_directory' in self.output:
if 'reference_reflectance_file' in self.infiles:
reference_file = self.infiles['reference_reflectance_file']
self.rfl_ref = reference_file.read_spectrum(row, col)
self.wl_ref = reference_file.wl
for i, x in enumerate(states):
# Calculate intermediate solutions
lamb_est, meas_est, path_est, S_hat, K, G = \
self.iv.forward_uncertainty(state_est, meas, geom)
plt.cla()
red = [0.7, 0.2, 0.2]
wl, fwhm = self.fm.calibration(x)
xmin, xmax = min(wl), max(wl)
fig = plt.subplots(1, 2, figsize=(10, 5))
plt.subplot(1, 2, 1)
meas_est = self.fm.calc_meas(x, geom)
for lo, hi in self.iv.windows:
idx = s.where(s.logical_and(wl > lo, wl < hi))[0]
p1 = plt.plot(wl[idx], meas[idx], color=red, linewidth=2)
p2 = plt.plot(wl, meas_est, color='k', linewidth=1)
plt.title("Radiance")
plt.title("Measurement (Scaled DN)")
ymax = max(meas) * 1.25
ymax = max(meas) + 0.01
ymin = min(meas) - 0.01
plt.text(500, ymax * 0.92, "Measured", color=red)
plt.text(500, ymax * 0.86, "Model", color='k')
plt.ylabel("$\mu$W nm$^{-1}$ sr$^{-1}$ cm$^{-2}$")
plt.ylabel("Intensity")
plt.xlabel("Wavelength (nm)")
plt.ylim([-0.001, ymax])
plt.ylim([ymin, ymax])
plt.xlim([xmin, xmax])
plt.subplot(1, 2, 2)
lamb_est = self.fm.calc_lamb(x, geom)
ymax = min(max(lamb_est) * 1.25, 0.10)
for lo, hi in self.iv.windows:
# black line
idx = s.where(s.logical_and(wl > lo, wl < hi))[0]
p2 = plt.plot(wl[idx], lamb_est[idx], 'k', linewidth=2)
ymax = max(max(lamb_est[idx] * 1.2), ymax)
# red line
if 'reference_reflectance_file' in self.infiles:
idx = s.where(
s.logical_and(self.wl_ref > lo,
self.wl_ref < hi))[0]
p1 = plt.plot(self.wl_ref[idx],
self.rfl_ref[idx],
color=red,
linewidth=2)
ymax = max(max(self.rfl_ref[idx] * 1.2), ymax)
# green and blue lines - surface components
if hasattr(self.fm.surface, 'components') and \
self.output['plot_surface_components']:
idx = s.where(
s.logical_and(self.fm.surface.wl > lo,
self.fm.surface.wl < hi))[0]
p3 = plt.plot(self.fm.surface.wl[idx],
self.fm.xa(x, geom)[idx],
'b',
linewidth=2)
for j in range(len(self.fm.surface.components)):
z = self.fm.surface.norm(
lamb_est[self.fm.surface.idx_ref])
mu = self.fm.surface.components[j][0] * z
plt.plot(self.fm.surface.wl[idx],
mu[idx],
'g:',
linewidth=1)
plt.text(500, ymax * 0.86, "Remote estimate", color='k')
if 'reference_reflectance_file' in self.infiles:
plt.text(500, ymax * 0.92, "In situ reference", color=red)
if hasattr(self.fm.surface, 'components') and \
self.output['plot_surface_components']:
plt.text(500, ymax * 0.80, "Prior mean state ", color='b')
plt.text(500,
ymax * 0.74,
"Surface components ",
color='g')
plt.ylim([-0.0010, ymax])
plt.xlim([xmin, xmax])
plt.title("Reflectance")
plt.title("Source Model")
plt.xlabel("Wavelength (nm)")
fn = self.output['plot_directory'] + ('/frame_%i.png' % i)
plt.savefig(fn)
plt.close()