def main(args):
if args['--sites'] is None:
sites = get_sites('PLUMBER_ext')
else:
try:
sites = get_sites(args['--sites'])
except:
sites = args['--sites'].split(',')
if args['all']:
if args['data']:
models = [os.path.basename(f) for f in glob('model_data/*')]
else:
models = [os.path.basename(f) for f in glob('source/models/*')]
else:
models = args['<model>']
if args['data']:
bad_sims = check_model_data(models, sites)
summary = "%d model with bad data out of %d models checked" % (
len(bad_sims), len(models))
if args['metrics']:
bad_sims = check_metrics(models, sites)
summary = "%d model with bad metrics out of %d models checked" % (
len(bad_sims), len(models))
if len(bad_sims) > 0:
if args['--delete-sims']:
for m, s in bad_sims:
os.remove('model_data/{m}/{m}_{s}.nc'.format(m=m, s=s))
summary += " and deleted sims"
if args['--delete-eval']:
for m, s in bad_sims:
os.remove(
'source/models/{m}/metrics/{m}_{s}_metrics.csv'.format(
m=m, s=s))
summary += " and deleted evals"
if args['--re-run']:
run_model_site_tuples_mp(bad_sims)
summary += " and re-run"
if args['--re-eval']:
[eval_simulation(t[0], t[1]) for t in bad_sims]
summary += " and re-evaluated"
logger.info(summary + ".")
return
def main(args):
if args['--sites'] is None:
sites = get_sites('PLUMBER_ext')
else:
sites = args['--sites'].split(',')
if args['all']:
if args['data']:
models = [os.path.basename(f) for f in glob('model_data/*')]
else:
models = [os.path.basename(f) for f in glob('source/models/*')]
else:
models = args['<model>']
if args['data']:
bad_sims = check_model_data(models, sites)
summary = "%d model with bad data out of %d models checked" % (
len(bad_sims), len(models))
if args['metrics']:
bad_sims = check_metrics(models, sites)
summary = "%d model with bad metrics out of %d models checked" % (
len(bad_sims), len(models))
if args['--re-run'] and len(bad_sims) > 0:
run_model_site_tuples_mp(bad_sims)
summary += " and re-run"
if args['--re-eval'] and len(bad_sims) > 0:
[eval_simulation(t[0], t[1]) for t in bad_sims]
summary += " and re-evaluated"
print(summary + ".")
return
def main_import_benchmark(name, site):
"""import a PLUMBER benchmark for all sites
:name: PLUMBER benchmark name
:site: plumber site name
"""
# Hacky solution just for PLUMBER benchmarks
print_good('Importing {n} data for: '.format(n=name))
if len(site) == 0:
datasets = get_sites('PLUMBER')
else:
datasets = site
for s in datasets:
print(s, end=', ', flush=True)
s_file = 'data/PALS/benchmarks/{n}/{n}_{s}Fluxnet.1.4.nc'.format(
n=name, s=s)
nc_path = get_sim_nc_path(name, s)
sim_data = xr.open_dataset(s_file)
fix_benchmark(sim_data, name, s)
# WARNING! over writes existing sim!
sim_data.to_netcdf(nc_path)
sim_data.close()
return
def fit_and_predict(name, dataset, years='2012-2013'):
"""Fit a benchmark to some PALS files, then generate an output matching a gridded dataset
"""
model = get_model(name)
met_vars = model.forcing_vars
flux_vars = ['Qle']
sites = get_sites('PLUMBER_ext')
years = [int(s) for s in years.split('-')]
print("Loading fluxnet data for %d sites" % len(sites))
met_data = pud.get_met_df(sites, met_vars, qc=True, name=True)
flux_data = pud.get_flux_df(sites, flux_vars, qc=True)
print("Fitting model {b} using {m} to predict {f}".format(n=name,
m=met_vars,
f=flux_vars))
model.fit(met_data, flux_data)
# prediction datasets
outdir = "{d}/gridded_benchmarks/{b}_{ds}".format(d=get_data_dir(),
n=name,
ds=dataset)
os.makedirs(outdir, exist_ok=True)
outfile_tpl = outdir + "/{b}_{d}_{v}_{y}.nc"
for year in range(*years):
print("Loading Forcing data for", year)
data = get_dataset_data(dataset, met_vars, year)
print("Predicting", year, end=': ', flush=True)
if "lag" in name:
result = predict_gridded(model,
data,
flux_vars,
datafreq=get_dataset_freq(dataset))
else:
result = predict_gridded(model, data, flux_vars)
xr_add_attributes(result, model, dataset, sites)
for fv in flux_vars:
filename = outfile_tpl.format(n=name, d=dataset, v=fv, y=year)
print("saving to ", filename)
result[[fv]].to_netcdf(filename,
encoding={fv: {
'dtype': 'float32'
}})
return