def _get_emulators(self, emulator_home, model="prosail"):
"""Based on the geometry, get the emulators. What could be simpler?"""
files = glob.glob("%s*.npz" % emulator_home)
emulator_search_dict = {}
for f in files:
emulator_search_dict[float(f.split("/")[-1].split("_")[0]),
float(f.split("/")[-1].split("_")[1]),
float(f.split("/")[-1].split("_")[2]),
float(f.split("/")[-1].split("_")[3])] = f
# So we have a dictionary inddexed by SZA, VZA and RAA and mapping to a filename
# Remove some weirdos...
emu_keys = np.array(emulator_search_dict.keys())
self.emulator = []
for i in xrange(len(self.metadata)):
e_sza = emu_keys[np.argmin(np.abs(emu_keys[:, 0] - self.sza[i])),
0]
e_vza = emu_keys[np.argmin(np.abs(emu_keys[:, 2] - self.vza[i])),
2]
e_saa = emu_keys[np.argmin(np.abs(emu_keys[:, 2] - self.saa[i])),
1]
e_vaa = emu_keys[np.argmin(np.abs(emu_keys[:, 3] - self.vaa[i])),
3]
print self.sza[i], e_sza, self.vza[i], e_vza, self.vaa[
i], e_vaa, self.saa[i], e_saa
the_emulator = "%.1f_%.1f_%.1f_%.1f_%s.npz" % (e_sza, e_saa, e_vza,
e_vaa, model)
print "Using emulator %s" % os.path.join(emulator_home,
the_emulator)
self.emulator.append(
gp_emulator.MultivariateEmulator(
dump=os.path.join(emulator_home, the_emulator)))
def _get_emulators(self, emulator_home, verbose=True):
"""Based on the geometry, get the emulators. What could be simpler?"""
n_geom = len(self.vza)
# Locate all available emulators...
files = glob.glob("%s*.npz" % emulator_home)
emulator_search_dict = {}
for f in files:
fs = f.replace(".npz", "")
try:
emulator_search_dict[ float(fs.split("/")[-1].split("_")[1]), \
float(fs.split("/")[-1].split("_")[2]),
float(fs.split("/")[-1].split("_")[3])] = f
except:
pass
# So we have a dictionary inddexed by SZA, VZA and RAA and mapping to a filename
raa = np.array(self.vaa) - np.array(self.saa)
emu_keys = np.array(emulator_search_dict.keys())
# Calculate the distance per dimension
emu_locs1 = np.array([(emu_keys[:, 0] - self.sza[i])**2
for i in xrange(n_geom)])
emu_locs2 = np.array([(emu_keys[:, 1] - self.vza[i])**2
for i in xrange(n_geom)])
emu_locs3 = np.array([(emu_keys[:, 2] - raa[i])**2
for i in xrange(n_geom)])
# The selected angles are given by the least distance
sel_emus1 = emu_locs1.argmin(axis=1)
sel_emus2 = emu_locs2.argmin(axis=1)
sel_emus3 = emu_locs3.argmin(axis=1)
# Stores the closest set of angles
all_angles = np.c_[emu_keys[sel_emus1, 0], emu_keys[sel_emus2, 1],
emu_keys[sel_emus3, 2]]
# We now go and select individual emulators. Since several observations might share
# the same emulator, we only load the emulator up once
emulators = {}
self.emu_keys = []
for i in xrange(n_geom):
the_key = tuple(all_angles[i, :].astype(np.int))
self.emu_keys.append(the_key)
if not emulators.has_key(the_key):
if verbose:
print "Reading (%d,%d,%d) from %s" % (
the_key[0], the_key[1], the_key[2],
emulator_search_dict[the_key])
emulators[the_key] = gp_emulator.MultivariateEmulator(
dump=emulator_search_dict[the_key])
self.emulator = emulators
def _load_emulator(emulator_file):
self.emulator = gp_emulator.MultivariateEmulator(dump=emulator_file)
def test_spectral_gp_read(datadir):
fname = datadir("prosail_000_000_0000.npz")
gp = gp_emulator.MultivariateEmulator(dump=fname)
assert np.allclose(gp.predict(gp.y_train[200, :])[0],
gp.X_train[200, :],
atol=0.05)
def do_emulator(sza, vza, raa, train_dir, validation_dir, plot_yn='n'):
plot_config()
files = glob.glob(os.path.join(train_dir,
"%d/%d/%d/*.p" % (sza, vza, raa)))
train_size = len(files)
params = np.zeros((train_size, 6))
#wvc, aot, alt, press, a, p
spectra = np.zeros((1, train_size, 5201))
#toa refl
for i, fich in enumerate(files):
xd = cPickle.load(open(fich, 'r'))
params[i, :] = [
xd['WVC'], xd['AOT'], xd['alt'], xd['press'], xd['a'], xd['p']
]
spectra[0, i, :] = xd['toa_refl']
files = glob.glob(
os.path.join(validation_dir, "%d/%d/%d/*.p" % (sza, vza, raa)))
if len(files) == 0:
raise IOError, "Are you sure this directory exists? (%s)" % \
(os.path.join ( validation_dir, "%d/%d/%d/" % ( sza, vza, raa)))
validation_size = len(files)
vparams = np.zeros((validation_size, 6))
#wvc, aot, alt, press
vspectra = np.zeros((1, validation_size, 5201))
# dlb_trans, atm_path, sph_albedo
for i, fich in enumerate(files):
xd = cPickle.load(open(fich, 'r'))
vparams[i, :] = [
xd['WVC'], xd['AOT'], xd['alt'], xd['press'], xd['a'], xd['p']
]
vspectra[0, i, :] = xd['toa_refl']
m1 = spectra[0, :, :].mean(axis=0)
M = [m1]
gp1 = gp_emulator.MultivariateEmulator(X=spectra[0, :, :],
y=params,
thresh=thresh,
n_tries=10)
# Save emulators
if not os.path.exists(os.path.join(train_dir, "emulators")):
os.mkdir(os.path.join(train_dir, "emulators"))
gp1.dump_emulator(
os.path.join(
train_dir, "emulators",
"libradtran_toa_refl_%02d_%02d_%03d.npz" % (sza, vza, raa)),
'toa_refl', sza, vza, raa)
vrmse1 = [(vspectra[0, i, :] - gp1.predict(vparams[i, :])[0])
for i in xrange(validation_size)]
V = [vrmse1]
if not os.path.exists(os.path.join(validation_dir, "diagnostics")):
os.mkdir(os.path.join(validation_dir, "diagnostics"))
if plot_yn.lower() == 'y':
fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(10, 5), sharex=True)
labels = ["TOA Reflectance"]
axs[1].plot(xd['lam'],
np.array(V[0]).std(axis=0),
'-',
lw=2,
label='std of rmse')
axs[1].plot(xd['lam'], M[0], '--', lw=2, label='mean training spectra')
axs[0].set_ylabel(labels[0])
pretty_axes(axs[1])
[ axs[0].plot ( xd['lam'], vspectra[0,j,:], "-k", alpha=0.5, lw=1, rasterized=True, label='validation spectra' ) \
for j in np.arange(0, validation_size,20)]
pretty_axes(axs[0])
axs[0].set_xlabel("Wavelength [nm]")
axs[1].set_xlabel("Wavelength [nm]")
fig.suptitle(
"%d training samples, %.1f%% variance kept.\n Validation on %d samples"
% (train_size, thresh * 100., validation_size) +
"\nSZA: %02d VZA: %02d RAA: %03d" % (sza, vza, raa))
axs[1].set_title("RMSE validation error")
axs[0].set_title("Validation plot")
plt.legend(loc='best')
plt.subplots_adjust(top=.8)
#plt.tight_layout()
piccy_fname = os.path.join(
os.path.join(
validation_dir, "diagnostics",
"validation_plot_%02d_%02d_%03d.pdf" % (sza, vza, raa)))
plt.savefig(piccy_fname, dpi=150, bbox_inches="tight")
#plt.savefig(piccy_fname.replace("pdf", "png"), dpi=150, bbox_inches="tight")
diagnostics_fname = os.path.join(
os.path.join(validation_dir, "diagnostics",
"diagnostics_%02d_%02d_%03d.txt" % (sza, vza, raa)))
vrmse1 = np.array(vrmse1)
rmse = "".join(["%15.5g" % f for f in vrmse1.mean(axis=0)])
rmse += "".join(["%15.5g" % f for f in vrmse1.std(axis=0)])
# Maybe we need to g
with open(diagnostics_fname, 'w') as fp:
fp.write("%02d %02d %03d %s" % (sza, vza, raa, rmse))
x['xs2'] = 0 #0.65
ds = np.load('model/params_%d.npz' % year)
lai_img = ds['lai']
kab_img = ds['kab']
scen_img = ds['scen']
brf_mod = np.zeros((6, lai_img.shape[1], lai_img.shape[2]))
brf_mod_0 = np.zeros((6, lai_img.shape[1], lai_img.shape[2]))
# emul_dir = '/home/ucfamc3/DATA/emulators/semidiscrete_3/lad2/'
# load emulator for nadir view
file_emu_0, nad_sza_0, nad_vza_0, nad_raa_0 = brdf_reg.find_nad_emul(
0, 0, 180, emul_dir)
em_0 = gp.MultivariateEmulator(dump=file_emu_0)
#for i, d in enumerate(doys):
# print 'doy:', doys[i], i
file_emu, nad_sza, nad_vza, nad_raa = brdf_reg.find_nad_emul(
sza, vza, np.abs(saa - vaa), emul_dir)
# print file_emu
em = gp.MultivariateEmulator(dump=file_emu)
for px in xrange(lai_img.shape[1]):
for py in xrange(lai_img.shape[2]):
# x['xlai'] = lai_img[doy-1, px, py]
# x['xkab'] = kab_img[doy-1, px, py]
# x['scen'] = scen_img[doy-1, px, py]