def visualise_albedos ( fluxnet_site, year ):
"""A function that will visualise albedo data for a particular year and site"""
observations, mask, bu, passer_snow = retrieve_albedo(year, fluxnet_site,
[0.05, 0.07])
passer = mask[:,1] == 1
doys = mask[:, 0]
plt.figure ( figsize=(12,6))
plt.plot ( doys[passer], observations[passer, 0], 'o', label="Visible Albedo")
plt.plot ( doys[passer], observations[passer, 1], 'o', label="Near-infrarred Albedo")
plt.vlines ( doys[passer], observations[passer, 0] + 1.96*bu[passer,0],
observations[passer, 0] - 1.96 * bu[passer, 0])
plt.vlines ( doys[passer], observations[passer, 1] + 1.96*bu[passer,1],
observations[passer, 1] - 1.96 * bu[passer, 1])
plt.legend(loc="best", numpoints=1, fancybox=True, shadow=True)
plt.ylabel("Bi-hemispherical reflectance [-]")
plt.xlabel("DoY/%d" % year )
plt.xlim ( 1, 368)
plt.ylim ( 0, 0.7)
ax1 = plt.gca()
ax1.figure.figimage(logo, 60, 60, alpha=.1, zorder=1)
# Hide the right and top spines
ax1.spines['right'].set_visible(False)
ax1.spines['top'].set_visible(False)
# Only show ticks on the left and bottom spines
ax1.yaxis.set_ticks_position('left')
ax1.xaxis.set_ticks_position('bottom')
def plot_albedos(fluxnet_site, year):
observations, mask, bu, passer_snow = retrieve_albedo(
year, fluxnet_site, [0.05, 0.07])
passer = mask[:, 1] == 1
doys = mask[:, 0]
plt.figure(figsize=(12, 6))
plt.plot(doys[passer],
observations[passer, 0],
'o',
label="Visible Albedo")
plt.plot(doys[passer],
observations[passer, 1],
'o',
label="Near-infrarred Albedo")
plt.vlines(doys[passer],
observations[passer, 0] + 1.96 * bu[passer, 0],
observations[passer, 0] - 1.96 * bu[passer, 0])
plt.vlines(doys[passer],
observations[passer, 1] + 1.96 * bu[passer, 1],
observations[passer, 1] - 1.96 * bu[passer, 1])
plt.legend(loc="best", numpoints=1, fancybox=True, shadow=True)
plt.ylabel("Bi-hemispherical reflectance [-]")
plt.xlabel("DoY/%d" % year)
plt.xlim(1, 368)
plt.ylim(0, 0.7)
ax1 = plt.gca()
ax1.figure.figimage(logo, 60, 60, alpha=.1, zorder=1)
# Hide the right and top spines
ax1.spines['right'].set_visible(False)
ax1.spines['top'].set_visible(False)
# Only show ticks on the left and bottom spines
ax1.yaxis.set_ticks_position('left')
ax1.xaxis.set_ticks_position('bottom')
def single_inversion ( year, site ):
n_tries = 2
observations, mask, bu, passer_snow = retrieve_albedo( year, site, [0.05, 0.07])
vis_emu_pkl = "helpers/tip_vis_emulator_real.pkl"
nir_emu_pkl = "helpers/tip_nir_emulator_real.pkl"
gp_vis = cPickle.load(open(vis_emu_pkl, 'r'))
gp_nir = cPickle.load(open(nir_emu_pkl, 'r'))
x0 = mu_prior
state = np.zeros((46,7))
for j,tstep in enumerate(np.arange(1, 366, 8)):
state[j,:] = mu_prior
if tstep in mask[:,0]:
i = mask[:,0] == tstep
is_ok = mask[i,1]
if is_ok == 1:
if passer_snow[i]:
mu = mu_prior_snow
inv_cov_prior = iprior_cov_snow
cov_prior = prior_cov_snow
else:
mu = mu_prior
inv_cov_prior = iprior_cov
cov_prior = prior_cov
cost_list = np.zeros(n_tries)
solutions = np.zeros((n_tries, 7))
for trial in xrange(n_tries):
if trial > 0:
while True:
x0 = np.random.multivariate_normal(mu, cov_prior, 1)
if np.all ( x0 > 0 ):
break
retval = tip_single_inversion(x0, observations[i, :].squeeze(), bu[i,:].squeeze(),
mu, inv_cov_prior, gp_vis, gp_nir)
cost_list[trial] = retval.fun
solutions[trial, :] = retval.x
best_sol = cost_list.argmin()
x0 = solutions[best_sol, :]
state[j,:] = solutions[best_sol, :]
return state
def single_inversion ( year, site ):
n_tries = 2
observations, mask, bu, passer_snow = retrieve_albedo( year, site, [0.05, 0.07])
vis_emu_pkl = "tip_vis_emulator_real.pkl"
nir_emu_pkl = "tip_nir_emulator_real.pkl"
gp_vis = cPickle.load(open(vis_emu_pkl, 'r'))
gp_nir = cPickle.load(open(nir_emu_pkl, 'r'))
x0 = mu_prior
state = np.zeros((46,7))
for j,tstep in enumerate(np.arange(1, 366, 8)):
state[j,:] = mu_prior
if tstep in mask[:,0]:
i = mask[:,0] == tstep
is_ok = mask[i,1]
if is_ok == 1:
if passer_snow[i]:
mu = mu_prior_snow
inv_cov_prior = iprior_cov_snow
cov_prior = prior_cov_snow
else:
mu = mu_prior
inv_cov_prior = iprior_cov
cov_prior = prior_cov
cost_list = np.zeros(n_tries)
solutions = np.zeros((n_tries, 7))
for trial in xrange(n_tries):
if trial > 0:
while True:
x0 = np.random.multivariate_normal(mu, cov_prior, 1)
if np.all ( x0 > 0 ):
break
retval = tip_single_inversion(x0, observations[i, :].squeeze(), bu[i,:].squeeze(),
mu, inv_cov_prior, gp_vis, gp_nir)
cost_list[trial] = retval.fun
solutions[trial, :] = retval.x
best_sol = cost_list.argmin()
x0 = solutions[best_sol, :]
state[j,:] = solutions[best_sol, :]
return state
def tip_inversion ( year, fluxnet_site, albedo_unc=[0.05, 0.07], green_leaves=False,
prior_type="TIP_standard",
vis_emu_pkl="tip_vis_emulator_real.pkl",
nir_emu_pkl="tip_nir_emulator_real.pkl", parallel=False,
n_tries=2, progressbar=None):
"""The JRC-TIP inversion using eoldas. This function sets up the
invesion machinery for a particular FLUXNET site and year (assuming
these are present in the database!)
Parameters
----------
year : int
The year
fluxnet_site: str
The code of the FLUXNET site (e.g. US-Bo1)
albedo_unc: list
A 2-element list, containg the relative uncertainty
prior_type: str
Not used yet
vis_emu_pkl: str
The emulator file for the visible band.
nir_emu_pkl: str
The emulator file for the NIR band.
n_tries: int
Number of restarts for the minimisation. Best one (e.g. lowest
cost) is chosen
parallel: bool
Whether to run the minimisation in parallel or not
progressbar: None or obj
A progressbar object to update
Returns
-------
Good stuff
"""
# We'll be running things in parallel, so a dummy function here is useful
def f (x_dict):
try:
state = copy.deepcopy (the_state)
retval = state.optimize(x_dict, do_unc=True)
cost= state.cost_history['global'][-1]
solution = retval
except Exception:
print("Exception in worker:")
traceback.print_exc()
raise
return cost, solution
# Start by setting up the state
the_state = StandardStateTIP ( state_config, state_grid,
optimisation_options=optimisation_options)
# Load and prepare the emulators for the TIP
gp_vis = cPickle.load(open(vis_emu_pkl, 'r'))
gp_nir = cPickle.load(open(nir_emu_pkl, 'r'))
# Retieve observatiosn and ancillary stuff from database
observations, mask, bu, passer_snow = retrieve_albedo ( year, fluxnet_site,
albedo_unc )
# Set up the observation operator
obsop = ObservationOperatorTIP ( state_grid, the_state, observations,
mask, [gp_vis, gp_nir], bu )
the_state.add_operator("Obs", obsop)
# Set up the prior
### prior = the_prior(the_state, prior_type )
prior = bernards_prior ( passer_snow, use_soil_corr=True,
green_leaves=green_leaves)
the_state.add_operator ("Prior", prior )
# Now, we will do the function minimisation with `n_tries` different starting
# points. We choose the one with the lowest cost...
# Is this needed?
#retval = single_inversion( year, fluxnet_site)
#x_dict = {}
#for i,k in enumerate ( ['omega_vis', 'd_vis', 'a_vis', 'omega_nir', 'd_nir', 'a_nir', 'lai']):
# x_dict[k] = retval[:,i]
x_tries = np.random.multivariate_normal( prior.mu,
np.array(np.linalg.inv(prior.inv_cov.todense())),
size=n_tries)
dicts = [ the_state._unpack_to_dict (x) for x in x_tries ]
pool = Pool()
results = pool.map ( f, dicts )
best_solution = np.array([ x[0] for x in results]).argmin()
print [ x[0] for x in results]
def regularised_tip_inversion ( year, fluxnet_site, gamma, x0, albedo_unc=[0.05, 0.07], green_leaves=False,
prior_type="TIP_standard",
vis_emu_pkl="tip_vis_emulator_real.pkl",
nir_emu_pkl="tip_nir_emulator_real.pkl", n_tries=2,
prior=None, progressbar=None):
"""The JRC-TIP inversion using eoldas. This function sets up the
invesion machinery for a particular FLUXNET site and year (assuming
these are present in the database!)
Parameters
----------
year : int
The year
fluxnet_site: str
The code of the FLUXNET site (e.g. US-Bo1)
albedo_unc: list
A 2-element list, containg the relative uncertainty
prior_type: str
Not used yet
vis_emu_pkl: str
The emulator file for the visible band.
nir_emu_pkl: str
The emulator file for the NIR band.
n_tries: int
Number of restarts for the minimisation. Best one (e.g. lowest
cost) is chosen
Returns
-------
Good stuff
"""
# The parallel dispatcher
def f (x_dict):
try:
state = copy.deepcopy (the_state)
retval = state.optimize(x_dict, do_unc=True)
cost= state.cost_history['global'][-1]
solution = retval
except Exception:
print("Exception in worker:")
traceback.print_exc()
raise
return cost, solution
# Start by setting up the state
the_state = StandardStateTIP ( state_config, state_grid,
optimisation_options=optimisation_options )
# Load and prepare the emulators for the TIP
gp_vis = cPickle.load(open(vis_emu_pkl, 'r'))
gp_nir = cPickle.load(open(nir_emu_pkl, 'r'))
# Retieve observatiosn and ancillary stuff from database
observations, mask, bu, passer_snow = retrieve_albedo ( year, fluxnet_site,
albedo_unc )
# Set up the observation operator
obsop = ObservationOperatorTIP ( state_grid, the_state, observations,
mask, [gp_vis, gp_nir], bu )
the_state.add_operator("Obs", obsop)
# Set up the prior
### prior = the_prior(the_state, prior_type )
if prior is None:
prior = bernards_prior(passer_snow, use_soil_corr=True,
green_leaves=green_leaves)
the_state.add_operator ("Prior", prior )
else:
the_state.add_operator("Prior", prior)
# Now, we will do the function minimisation with `n_tries` different starting
# points. We choose the one with the lowest cost...
smoother = eoldas_ng.TemporalSmoother ( state_grid, gamma, required_params =
["omega_vis", "d_vis", "a_vis",
"omega_nir", "d_nir", "a_nir", "lai"] )
the_state.add_operator ( "Smooth", smoother)
x_tries = np.random.multivariate_normal( prior.mu,
np.array(np.linalg.inv(prior.inv_cov.todense())),
size=n_tries)
dicts = [ the_state._unpack_to_dict (x) for x in x_tries ]
pool = Pool()
results = pool.map ( f, dicts )
#####for i in xrange(n_tries):
#####if n_tries > 1:
#####x0 = np.random.multivariate_normal( prior.mu, np.array(np.linalg.inv(prior.inv_cov.todense())))
#####x_dict = the_state._unpack_to_dict ( x0 )
#####retval = the_state.optimize(x_dict, do_unc=True)
#####results.append ( ( the_state.cost_history['global'][-1],
#####retval ) )
#####if progressbar is not None:
#####progressbar.value = progressbar.value + 1
best_solution = np.array([ x[0] for x in results]).argmin()
print [ x[0] for x in results]
def tip_inversion(year,
fluxnet_site,
albedo_unc=[0.05, 0.07],
green_leaves=False,
prior_type="TIP_standard",
vis_emu_pkl="tip_vis_emulator_real.pkl",
nir_emu_pkl="tip_nir_emulator_real.pkl",
parallel=False,
n_tries=2,
progressbar=None):
"""The JRC-TIP inversion using eoldas. This function sets up the
invesion machinery for a particular FLUXNET site and year (assuming
these are present in the database!)
Parameters
----------
year : int
The year
fluxnet_site: str
The code of the FLUXNET site (e.g. US-Bo1)
albedo_unc: list
A 2-element list, containg the relative uncertainty
prior_type: str
Not used yet
vis_emu_pkl: str
The emulator file for the visible band.
nir_emu_pkl: str
The emulator file for the NIR band.
n_tries: int
Number of restarts for the minimisation. Best one (e.g. lowest
cost) is chosen
parallel: bool
Whether to run the minimisation in parallel or not
progressbar: None or obj
A progressbar object to update
Returns
-------
Good stuff
"""
# We'll be running things in parallel, so a dummy function here is useful
def f(x_dict):
try:
state = copy.deepcopy(the_state)
retval = state.optimize(x_dict, do_unc=True)
cost = state.cost_history['global'][-1]
solution = retval
except Exception:
print("Exception in worker:")
traceback.print_exc()
raise
return cost, solution
# Start by setting up the state
the_state = StandardStateTIP(state_config,
state_grid,
optimisation_options=optimisation_options)
# Load and prepare the emulators for the TIP
gp_vis = cPickle.load(open(vis_emu_pkl, 'r'))
gp_nir = cPickle.load(open(nir_emu_pkl, 'r'))
# Retieve observatiosn and ancillary stuff from database
observations, mask, bu, passer_snow = retrieve_albedo(
year, fluxnet_site, albedo_unc)
# Set up the observation operator
obsop = ObservationOperatorTIP(state_grid, the_state, observations, mask,
[gp_vis, gp_nir], bu)
the_state.add_operator("Obs", obsop)
# Set up the prior
### prior = the_prior(the_state, prior_type )
prior = bernards_prior(passer_snow,
use_soil_corr=True,
green_leaves=green_leaves)
the_state.add_operator("Prior", prior)
# Now, we will do the function minimisation with `n_tries` different starting
# points. We choose the one with the lowest cost...
# Is this needed?
#retval = single_inversion( year, fluxnet_site)
#x_dict = {}
#for i,k in enumerate ( ['omega_vis', 'd_vis', 'a_vis', 'omega_nir', 'd_nir', 'a_nir', 'lai']):
# x_dict[k] = retval[:,i]
x_tries = np.random.multivariate_normal(
prior.mu,
np.array(np.linalg.inv(prior.inv_cov.todense())),
size=n_tries)
dicts = [the_state._unpack_to_dict(x) for x in x_tries]
pool = Pool()
results = pool.map(f, dicts)
best_solution = np.array([x[0] for x in results]).argmin()
print[x[0] for x in results]
print "Chosen cost: %g" % results[best_solution][0]
# This is needed to do the forward calculations
x = the_state.pack_from_dict(results[best_solution][1]['real_map'])
_ = the_state.cost(x)
return results[best_solution][1], the_state, obsop
def regularised_tip_inversion(year,
fluxnet_site,
gamma,
x0,
albedo_unc=[0.05, 0.07],
green_leaves=False,
prior_type="TIP_standard",
vis_emu_pkl="tip_vis_emulator_real.pkl",
nir_emu_pkl="tip_nir_emulator_real.pkl",
n_tries=2,
prior=None,
progressbar=None):
"""The JRC-TIP inversion using eoldas. This function sets up the
invesion machinery for a particular FLUXNET site and year (assuming
these are present in the database!)
Parameters
----------
year : int
The year
fluxnet_site: str
The code of the FLUXNET site (e.g. US-Bo1)
albedo_unc: list
A 2-element list, containg the relative uncertainty
prior_type: str
Not used yet
vis_emu_pkl: str
The emulator file for the visible band.
nir_emu_pkl: str
The emulator file for the NIR band.
n_tries: int
Number of restarts for the minimisation. Best one (e.g. lowest
cost) is chosen
Returns
-------
Good stuff
"""
# The parallel dispatcher
def f(x_dict):
try:
state = copy.deepcopy(the_state)
retval = state.optimize(x_dict, do_unc=True)
cost = state.cost_history['global'][-1]
solution = retval
except Exception:
print("Exception in worker:")
traceback.print_exc()
raise
return cost, solution
# Start by setting up the state
the_state = StandardStateTIP(state_config,
state_grid,
optimisation_options=optimisation_options)
# Load and prepare the emulators for the TIP
gp_vis = cPickle.load(open(vis_emu_pkl, 'r'))
gp_nir = cPickle.load(open(nir_emu_pkl, 'r'))
# Retieve observatiosn and ancillary stuff from database
observations, mask, bu, passer_snow = retrieve_albedo(
year, fluxnet_site, albedo_unc)
# Set up the observation operator
obsop = ObservationOperatorTIP(state_grid, the_state, observations, mask,
[gp_vis, gp_nir], bu)
the_state.add_operator("Obs", obsop)
# Set up the prior
### prior = the_prior(the_state, prior_type )
if prior is None:
prior = bernards_prior(passer_snow,
use_soil_corr=True,
green_leaves=green_leaves)
the_state.add_operator("Prior", prior)
else:
the_state.add_operator("Prior", prior)
# Now, we will do the function minimisation with `n_tries` different starting
# points. We choose the one with the lowest cost...
smoother = eoldas_ng.TemporalSmoother(state_grid,
gamma,
required_params=[
"omega_vis", "d_vis", "a_vis",
"omega_nir", "d_nir", "a_nir",
"lai"
])
the_state.add_operator("Smooth", smoother)
x_tries = np.random.multivariate_normal(
prior.mu,
np.array(np.linalg.inv(prior.inv_cov.todense())),
size=n_tries)
dicts = [the_state._unpack_to_dict(x) for x in x_tries]
pool = Pool()
results = pool.map(f, dicts)
#####for i in xrange(n_tries):
#####if n_tries > 1:
#####x0 = np.random.multivariate_normal( prior.mu, np.array(np.linalg.inv(prior.inv_cov.todense())))
#####x_dict = the_state._unpack_to_dict ( x0 )
#####retval = the_state.optimize(x_dict, do_unc=True)
#####results.append ( ( the_state.cost_history['global'][-1],
#####retval ) )
#####if progressbar is not None:
#####progressbar.value = progressbar.value + 1
best_solution = np.array([x[0] for x in results]).argmin()
print[x[0] for x in results]
print "Chosen cost: %g" % results[best_solution][0]
# This is needed to do the forward calculations
x = the_state.pack_from_dict(results[best_solution][1]['real_map'])
_ = the_state.cost(x)
return results[best_solution][1], the_state, obsop