def main_import_sim(name, site, sim_file):
"""import a PLUMBER benchmark for all sites
:name: PLUMBER benchmark name
:site: plumber site name
"""
# Hacky solution just for PLUMBER benchmarks
print_good('Importing {n} data for: {s}'.format(n=name, s=site))
nc_path = get_sim_nc_path(name, site)
data_vars = ['Qh', 'Qle', 'NEE']
with xr.open_dataset(sim_file) as ds:
d_vars = [v for v in data_vars if v in ds]
sim_data = ds[d_vars].copy(deep=True)
if name == 'CHTESSEL':
print("Inverting CHTESSEL and adding data")
# F*****g inverted, and
sim_data = -sim_data
# missing most of the last f*****g day, and
tsteps = ds.dims['time']
complete_tsteps = 48 * int(np.ceil(tsteps / 48))
missing_tsteps = complete_tsteps - tsteps
new_data = (sim_data.isel(time=slice(-missing_tsteps, None)).copy(
deep=True))
# f*****g off-set by an hour.
with get_flux_data([site])[site] as ds:
site_time = ds.time.values.flat.copy()
sim_data['time'] = site_time[:tsteps]
new_data['time'] = site_time[tsteps:]
sim_data = xr.concat([sim_data, new_data], dim='time')
if name == 'ORCHIDEE.trunk_r1401':
sim_data.rename(dict(time_counter='time'), inplace=True)
# Stores data for all vegetation types
# NEE_veget_index = np.where(sim_data.NEE
# .isel(time_counter=0, lat=0, lon=0)
# .values.flat > 0)[0][0]
# sim_data['NEE'] = sim_data['NEE'].isel(veget=NEE_veget_index)
print("Flattening veg in ORCHIDEE")
sim_data['NEE'] = sim_data['NEE'].sum(axis=1) # Sum over veget axis
if site == 'Espirra':
print("Deleting a year of data at Espirra for Orchidee")
sim_data = sim_data.isel(time=slice(70128))
# WARNING! over writes existing sim!
print('Writing to', nc_path)
sim_data.to_netcdf(nc_path)
sim_data.close()
return
def main_import_benchmark(name, site):
"""import a PLUMBER benchmark for all sites
:name: PLUMBER benchmark name
:site: plumber site name
"""
# Hacky solution just for PLUMBER benchmarks
print_good('Importing {n} data for: '.format(n=name))
if len(site) == 0:
datasets = get_sites('PLUMBER')
else:
datasets = site
for s in datasets:
print(s, end=', ', flush=True)
s_file = 'data/PALS/benchmarks/{n}/{n}_{s}Fluxnet.1.4.nc'.format(
n=name, s=s)
nc_path = get_sim_nc_path(name, s)
sim_data = xr.open_dataset(s_file)
fix_benchmark(sim_data, name, s)
# WARNING! over writes existing sim!
sim_data.to_netcdf(nc_path)
sim_data.close()
return
def eval_simulation(name, site, sim_file=None, plots=False, fix_closure=True,
qc=True, overwrite=False):
"""Main function for evaluating an existing simulation.
Copies simulation data to source directory.
TODO: skip running if cached, for easier page regeneration
:name: name of the model
:site: PALS site name to run the model at
:sim_file: Path to simulation netcdf. Only required if simulation is at a non-standard place.
"""
args = locals()
args_str = '\n'.join([k + ': ' + str(args[k]) for k in sorted(args.keys())])
logger = setup_logger(__name__, 'logs/eval/{m}/{s}/{m}_{s}.log'.format(m=name, s=site))
logger.info("Evaluating model.\nArgs:\n{a}".format(a=args_str))
nc_path = get_sim_nc_path(name, site)
if sim_file is None:
filename = nc_path
else:
filename = sim_file
eval_path = 'source/models/{n}/metrics/{n}_{s}_metrics.csv'.format(n=name, s=site)
if not overwrite:
if os.path.exists(filename) and os.path.exists(eval_path) and \
os.path.getmtime(filename) > os.path.getmtime(eval_path):
logger.warning("Overwriting evaluation file because simulation is newer")
else:
logger.warning("Evaluation file already exists, skipping")
return
try:
sim_data = xr.open_dataset(filename)
except (OSError, RuntimeError) as e:
logger.error("Sim file ({f}) doesn't exist. What are you doing? {e}".format(f=filename, e=e))
return
if sim_file is not None:
logger.warning("Overwriting existing sim!")
sim_data.to_netcdf(nc_path)
flux_data = get_flux_data([site], fix_closure=fix_closure)[site]
evaluate_simulation(sim_data, flux_data, name, site=site, qc=qc)
if plots:
diagnostic_plots(sim_data, flux_data, name, site=site)
sim_data.close()
return
def eval_simulation(name,
site,
sim_file=None,
plots=False,
fix_closure=True,
qc=True):
"""Main function for evaluating an existing simulation.
Copies simulation data to source directory.
TODO: skip running if cached, for easier page regeneration
:name: name of the model
:site: PALS site name to run the model at
:sim_file: Path to simulation netcdf
"""
nc_path = get_sim_nc_path(name, site)
if sim_file is None:
filename = nc_path
else:
filename = sim_file
try:
sim_data = xr.open_dataset(filename)
except (OSError, RuntimeError) as e:
print_bad(
"Sim file ({f}) doesn't exist. What are you doing? {e}".format(
f=filename, e=e))
return
if sim_file is not None:
# WARNING! over writes existing sim!
sim_data.to_netcdf(nc_path)
flux_data = get_flux_data([site], fix_closure=fix_closure)[site]
evaluate_simulation(sim_data, flux_data, name, qc=qc)
if plots:
diagnostic_plots(sim_data, flux_data, name)
sim_data.close()
return