def prep_training_N_target(profile, sub=None):
base_dir = get_main_dir()
# path to input data
fname = os.path.join(
os.path.join(
os.path.join(os.path.join(base_dir, 'input'), 'calibrations'),
'idealised'), 'training_x.csv')
df1, __ = read_csv(fname)
# path to output data from the reference model
fname = os.path.join(
os.path.join(
os.path.join(os.path.join(base_dir, 'input'), 'calibrations'),
'idealised'), 'training_%s_y.csv' % (profile))
df2, __ = read_csv(fname)
# add the soil moisture profile to the input data
df1['sw'], df1['Ps'] = soil_water(df1, profile)
df1['Ps_pd'] = df1['Ps'].copy() # daily pre-dawn soil water potentials
df1['Ps_pd'].where(df1['PPFD'] <= 50., np.nan, inplace=True)
# fix the wind speed
df1['u'] = df1['u'].iloc[0]
# non time-sensitive: last valid value propagated until next valid
df1.fillna(method='ffill', inplace=True)
# drop everything below min threshold for photosynthesis and reindex
Y = np.asarray(df2['gs(std)'][df1['PPFD'] > 50.]) * 1000. # mmol m-2 s-1
X = df1[df1['PPFD'] > 50.]
X.reset_index(inplace=True, drop=True)
# add Rnet to the input (no ET or soil albedo feedbacks, this can be done)
X['Rnet'] = net_radiation(X)
X['scale2can'] = 1.
if sub is not None: # randomly subsample one week out of the data
X, Y = subsample(X, Y, sub)
return X, Y
def check_X_Y(swaters):
base_dir = get_main_dir()
# check that the 4 week forcing file exists
fname1 = os.path.join(
os.path.join(
os.path.join(os.path.join(base_dir, 'input'), 'calibrations'),
'idealised'), 'training_x.csv')
if not os.path.isfile(fname1): # create file if it doesn't exist
params = InForcings().defparams
params.doy = random.randrange(92, 275) # random day within GS
InForcings().run(fname1, params, Ndays=7 * 4)
for profile in swaters:
# check that the output file from the reference model exists
fname2 = os.path.join(
os.path.join(
os.path.join(os.path.join(base_dir, 'input'), 'calibrations'),
'idealised'), 'training_%s_y.csv' % (profile))
if not os.path.isfile(fname2):
df1, __ = read_csv(fname1)
# add the soil moisture profile to the input data
df1['sw'], df1['Ps'] = soil_water(df1, profile)
df1['Ps_pd'] = df1['Ps'].copy() # pre-dawn soil water potentials
df1['Ps_pd'].where(df1['PPFD'] <= 50., np.nan, inplace=True)
# fixed value for the wind speed
df1['u'] = df1['u'].iloc[0]
# non time-sensitive: last valid value propagated until next valid
df1.fillna(method='ffill', inplace=True)
__ = hrun(fname2,
df1,
len(df1.index),
'Farquhar',
models=['Medlyn'],
inf_gb=True)
return
def build_calibrated_forcing(training):
base_dir = get_main_dir() # working paths
# forcing file used to calibrate the models
fname = os.path.join(os.path.join(os.path.join(os.path.join(base_dir,
'input'), 'calibrations'), 'obs_driven'),
'%s_x.csv' % (training))
df1, columns = read_csv(fname)
# file containing the best calibrated params
fname = os.path.join(os.path.join(os.path.join(os.path.join(base_dir,
'output'), 'calibrations'), 'obs_driven'),
'best_fit.csv')
df2 = (pd.read_csv(fname, header=[0]).dropna(axis=0, how='all')
.dropna(axis=1, how='all').squeeze())
df2 = df2[df2['training'] == training]
# attribute the first (and second and third) parameter(s)
for i in df2.index:
df1.loc[0, df2.loc[i, 'p1']] = df2.loc[i, 'v1']
if not pd.isnull(df2.loc[i, 'v2']):
df1.loc[0, df2.loc[i, 'p2']] = df2.loc[i, 'v2']
if not pd.isnull(df2.loc[i, 'v3']):
df1.loc[0, df2.loc[i, 'p3']] = df2.loc[i, 'v3']
# save the forcing file containing the calibrated params
df1.columns = columns # original columns
df1.drop([('Tleaf', '[deg C]')], axis=1, inplace=True) # drop Tleaf
df1.to_csv(os.path.join(os.path.join(os.path.join(os.path.join(base_dir,
'input'), 'simulations'), 'obs_driven'),
'%s_calibrated.csv' % (training)), index=False, na_rep='',
encoding='utf-8')
return
###############################################################################
base_dir = get_main_dir()
# path to input data
Pmin = 1
fname = os.path.join(
os.path.join(os.path.join(os.path.join(base_dir, 'input'), 'simulations'),
'idealised'), 'sensitivity_mtx_%sMPa.csv' % (str(Pmin)))
# create the pbm for the Sobol sensitivity analysis
fname1 = os.path.join(
os.path.join(os.path.join(os.path.join(base_dir, 'input'), 'calibrations'),
'idealised'), 'training_x.csv')
df1, columns = read_csv(fname1) # load training data
# get the bounds for the different variables to look at
variables = ['PPFD', 'Tair', 'VPD', 'CO2', 'Ps']
PPFD = [50., 2500.] # umol m-2 s-1
Tair = [2., 40.] # degC
VPD = [0.1, 10.] # kPa
CO2 = [250. * 101.325 / 1000., 900. * 101.325 / 1000.] # Pa, 250 - 900 ppm
Ps = [-Pmin, df1['Psie'].iloc[0]]
bounds = [PPFD, Tair, VPD, CO2, Ps]
# define the sensitivity problem
problem = {'num_vars': len(variables), 'names': variables, 'bounds': bounds}
if not os.path.isfile(fname): # generate the sensitivity inputs
N = 84000 # Saltelli mtx obtained via cross-sampling method
print(fname)
fig.savefig(fname, dpi=600)
plt.close()
print('done')
return
###############################################################################
# working paths
base_dir = get_main_dir()
ipath = os.path.join(os.path.join(os.path.join(base_dir, 'input'),
'simulations'), 'idealised')
df, __ = read_csv(os.path.join(ipath, 'wet_calibration.csv'))
# soil moisture profile
df['sw'] = df['theta_sat']
df.fillna(method='ffill', inplace=True)
df['sw'], df['Ps'] = soil_water(df)
# week 1, at some point in the middle of the week
df = df[df['doy'] >= df['doy'].iloc[0] + 7 * 3] #4]
# run the models
models = ['WUE', 'ProfitMax', 'CGain', 'ProfitMax2', 'LeastCost']
# actual form
df1 = hrun(None, df, 48, 'Farquhar', models=models,
resolution='low', inf_gb=True)
for training in site_spp: # loop over the site x spp combinations
if not os.path.isfile(os.path.join(ipath,
'%s_calibrated.csv' % (training))):
build_calibrated_forcing(training)
for file in os.listdir(ipath): # loop over all the possibilities
# output dir paths
ofdir = os.path.join(ipath.replace('input', 'output'), 'all_site_spp')
if not os.path.isdir(ofdir):
os.makedirs(ofdir)
# load input data into a dataframe
df1, __ = read_csv(os.path.join(ipath, file))
df1.fillna(method='ffill', inplace=True)
# add the necessary extra variables
df1['Ps_pd'] = df1['Ps'].copy()
df1['sw'] = 0. # add sw (not used) or it won't run
df1['scale2can'] = 1.
# run the models
fname = os.path.join(ofdir, '%s.csv' % (file.split('_calibrated')[0]))
if not os.path.isfile(fname): # create file if it doesn't exist
df2 = hrun(fname, df1, len(df1.index), 'Farquhar',
models=['Medlyn', 'Tuzet', 'SOX12', 'WUE', 'CMax',
'ProfitMax', 'CGain', 'ProfitMax2', 'LeastCost',
'CAP', 'MES'], resolution='high')
###############################################################################
# first, activate user defined rendering options
plt_setup()
base_dir = get_main_dir() # dir paths
ifdir = os.path.join(
os.path.join(os.path.join(base_dir, 'input'), 'simulations'), 'idealised')
ofdir = os.path.join(
os.path.join(os.path.join(base_dir, 'output'), 'simulations'), 'idealised')
# path to input data
fname1 = os.path.join(ifdir, 'wet_calibration.csv')
df1, __ = read_csv(fname1)
# initialise soil moisture forcings
df1['sw'] = df1['theta_sat']
df1.fillna(method='ffill', inplace=True)
# plot the atmospheric forcings
figdir = os.path.join(os.path.join(base_dir, 'output'), 'plots')
figname = os.path.join(figdir, 'training_forcing_soil_moisture.png')
if not os.path.isfile(figname):
plot_forcings(df1, figname)
# plot the calibration targets
fname2 = os.path.join(ifdir.replace('simulation', 'calibration'),
'training_wet_y.csv')
def obs_calibs(df1, df2, figname):
fig = plt.figure(figsize=(6.5, 8.))
gs = fig.add_gridspec(nrows=96, ncols=16, hspace=0.3, wspace=0.2)
ax2 = fig.add_subplot(gs[52:, 6:]) # conductance data
ipath = os.path.join(
os.path.join(os.path.join(get_main_dir(), 'input'), 'simulations'),
'obs_driven')
labels = []
for i, what in enumerate(df1['site_spp'].unique().dropna()):
if i < 13:
nrow = int(i / 4) * 16
ncol = (i % 4) * 4
ax1 = fig.add_subplot(gs[nrow:nrow + 16, ncol:ncol + 4])
else:
nrow += 16
ax1 = fig.add_subplot(gs[nrow:nrow + 16, :4])
sub = df1.copy()[df1['site_spp'] == what]
sub = sub.select_dtypes(exclude=['object', 'category'])
sub = sub[sub['Pleaf'] > -9999.]
sub['gs'] /= sub['gs'].max()
for day in sub['doy'].unique():
mask = sub['doy'] == day
plot_obs(ax1, sub['Pleaf'][mask], sub['gs'][mask])
x0, x1, obs_popt = fit_Tuzet(sub)
x = np.linspace(sub['Pleaf'].max(), sub['Pleaf'].min(), 500)
ax1.plot(x, fsig_tuzet(x, obs_popt[0], obs_popt[1]), 'k', zorder=30)
ax1.vlines(x0, 0., 1., linestyle=':')
ax1.vlines(x1, 0., 1., linestyle=':')
# get the integrated VC given by the obs and site params
ref, __ = read_csv(os.path.join(ipath, '%s_calibrated.csv' % (what)))
b, c = Weibull_params(ref.iloc[0])
int_VC = np.zeros(len(sub))
for j in range(len(sub)):
int_VC[j], __ = quad(f,
sub['Pleaf'].iloc[j],
sub['Ps'].iloc[j],
args=(b, c))
plot_obs(ax2, i, np.log(sub['E'] / int_VC), which='kmax')
# subplot titles (including labelling)
what = what.split('_')
species = r'\textit{%s %s}' % (what[-2], what[-1])
labels += [r'\textit{%s. %s}' % (what[-2][0], what[-1])]
if 'Quercus' in what:
species += ' (%s)' % (what[0][0])
labels[-1] += ' (%s)' % (what[0][0])
txt = ax1.annotate(r'\textbf{(%s)} %s' %
(string.ascii_lowercase[i], species),
xy=(0.025, 0.98),
xycoords='axes fraction',
ha='left',
va='top')
txt.set_bbox(
dict(boxstyle='round,pad=0.1', fc='w', ec='none', alpha=0.8))
# format axes ticks
ax1.xaxis.set_major_locator(mpl.ticker.NullLocator())
if (i == 13) or ((ncol > 0) and (nrow == 32)):
render_xlabels(ax1, r'$\Psi_{l}$', 'MPa')
if ncol == 0:
ax1.yaxis.set_major_locator(mpl.ticker.MaxNLocator(3))
ax1.yaxis.set_major_formatter(
mpl.ticker.FormatStrFormatter('%.1f'))
ax1.set_ylabel(r'$g_{s, norm}$')
else:
ax1.yaxis.set_major_locator(mpl.ticker.MaxNLocator(3))
ax1.set_yticklabels([])
ax2.annotate(r'\textbf{(%s)}' % (string.ascii_lowercase[i + 1]),
xy=(0.05, 0.98),
xycoords='axes fraction',
ha='right',
va='top')
# add max conductance parameter values
params, models = get_calib_kmax(df2)
params = np.asarray(params)
locs = np.arange(len(df1['site_spp'].unique()))
# update colour list
colours = ([
'#6023b7', '#af97c5', '#009231', '#6b3b07', '#ff8e12', '#ffe020',
'#f10c80', '#ffc2cd'
]) * len(params)
for i in range(params.shape[1]):
if i < 8:
ax2.scatter(locs,
params[:, i],
s=50,
linewidths=0.25,
c=colours[i],
alpha=0.9,
label=models[0][i],
zorder=4)
else:
ax2.scatter(locs,
params[:, i],
s=50,
linewidths=0.25,
c=colours[i],
alpha=0.9,
zorder=4)
# tighten the subplot
ax2.set_xlim(locs[0] - 0.8, locs[-1] + 0.8)
ax2.set_ylim(np.log(0.025) - 0.1, np.log(80.))
# ticks
ax2.set_xticks(locs + 0.5)
ax2.set_xticklabels(labels, ha='right', rotation=40)
ax2.xaxis.set_tick_params(length=0.)
yticks = [0.025, 0.25, 1, 5, 25, 75]
ax2.set_yticks([np.log(e) for e in yticks])
ax2.set_yticklabels(yticks)
render_ylabels(ax2, r'k$_{max}$', 'mmol m$^{-2}$ s$^{-1}$ MPa$^{-1}$')
handles, labels = ax2.get_legend_handles_labels()
labels[3] = 'SOX$_\mathrm{\mathsf{opt}}$'
ax2.legend(handles,
labels,
ncol=3,
labelspacing=1. / 3.,
columnspacing=0.5,
loc=3)
# save
fig.savefig(figname)
'simulations'), 'idealised')
for training in trainings:
if not os.path.isfile(os.path.join(ipath,
'%s_calibration.csv' % (training))):
build_calibrated_forcing(training)
combis = [e for e in combis if ((e[0] == e[1]) or (e[2] == 'insample'))]
for combi in combis: # loop over all the possibilities
xpe = '%s_%s_%s' % (combi[2], combi[1], combi[0])
# load input data into a dataframe
df, __ = read_csv(os.path.join(ipath, '%s_calibration.csv' % (combi[0])))
# output dir paths
ofdir = os.path.join(ipath.replace('input', 'output'), 'multivar_change')
if xpe in univar_xpes:
ofdir = ofdir.replace('multivar_change', 'univar_change')
if not os.path.isdir(ofdir):
os.makedirs(ofdir)
# how should the atm forcing data change?
df1 = df.copy() # reset the df so as to not keep previous changes
if combi[2] == 'highD':
df1['VPD'] *= 2.