def run_simulation_mp(name,
site,
no_mp=False,
multivariate=False,
overwrite=False,
fix_closure=True):
"""Multi-processor run handling."""
# TODO: refactor to work with above caller.
model = get_model(name)
if site in ['all', 'PLUMBER_ext', 'PLUMBER']:
logger.info('Running {n} at {s} sites'.format(n=name, s=site))
datasets = get_sites(site)
if no_mp:
for s in datasets:
run_simulation(model, name, s, multivariate, overwrite,
fix_closure)
else:
f_args = [(model, name, s, multivariate, overwrite, fix_closure)
for s in datasets]
ncores = get_suitable_ncores()
if site is not 'debug' and hasattr(model, 'memory_requirement'):
ncores = max(
1,
int((psutil.virtual_memory().total / 2) //
model.memory_requirement))
logger.info("Running on %d core(s)" % ncores)
with Pool(ncores) as p:
p.starmap(run_simulation, f_args)
else:
run_simulation(model, name, site, multivariate, overwrite, fix_closure)
return
def fit_and_predict(name, dataset, years='2012-2013'):
"""Fit a benchmark to some PALS files, then generate an output matching a gridded dataset
"""
model = get_model(name)
met_vars = model.forcing_vars
flux_vars = ['Qle']
sites = get_sites('PLUMBER_ext')
years = [int(s) for s in years.split('-')]
print("Loading fluxnet data for %d sites" % len(sites))
met_data = pud.get_met_df(sites, met_vars, qc=True, name=True)
flux_data = pud.get_flux_df(sites, flux_vars, qc=True)
print("Fitting model {b} using {m} to predict {f}".format(n=name,
m=met_vars,
f=flux_vars))
model.fit(met_data, flux_data)
# prediction datasets
outdir = "{d}/gridded_benchmarks/{b}_{ds}".format(d=get_data_dir(),
n=name,
ds=dataset)
os.makedirs(outdir, exist_ok=True)
outfile_tpl = outdir + "/{b}_{d}_{v}_{y}.nc"
for year in range(*years):
print("Loading Forcing data for", year)
data = get_dataset_data(dataset, met_vars, year)
print("Predicting", year, end=': ', flush=True)
if "lag" in name:
result = predict_gridded(model,
data,
flux_vars,
datafreq=get_dataset_freq(dataset))
else:
result = predict_gridded(model, data, flux_vars)
xr_add_attributes(result, model, dataset, sites)
for fv in flux_vars:
filename = outfile_tpl.format(n=name, d=dataset, v=fv, y=year)
print("saving to ", filename)
result[[fv]].to_netcdf(filename,
encoding={fv: {
'dtype': 'float32'
}})
return
def run_model_site_tuples_mp(tuples_list,
no_mp=False,
multivariate=False,
overwrite=False,
fix_closure=True):
"""Run (model, site) pairs"""
seen = set()
all_names = [
t[0] for t in tuples_list if not (t[0] in seen or seen.add(t[0]))
]
all_models = {n: get_model(n) for n in all_names}
f_args = [(all_models[t[0]], t[0], t[1], multivariate, overwrite,
fix_closure) for t in tuples_list]
if no_mp:
[run_simulation(*args) for args in f_args]
else: # multiprocess
ncores = get_suitable_ncores()
# TODO: Deal with memory requirement?
with Pool(ncores) as p:
p.starmap(run_simulation, f_args)
def model_site_rst_format(name, site, eval_text, plot_files):
"""format all the datas into an rst!
"""
date = dt.isoformat(dt.now().replace(microsecond=0), sep=' ')
plots_text = ''
for group in sorted(plot_files):
plots_text += "{g}\n".format(g=group)
plots_text += "^" * len(group) + "\n\n"
plots_text += '\n\n'.join([
".. image :: {file}\n :width: 200px".format(file=f)
for f in sorted(plot_files[group])
])
plots_text += '\n\n'
title = '{name} at {site}'.format(name=name, site=site)
title += '\n' + '=' * len(title)
try:
model = get_model(name)
except:
model = None
if hasattr(model, "_var_lags"):
forcing_vars = model._var_lags
elif hasattr(model, "forcing_vars"):
forcing_vars = model.forcing_vars
else:
forcing_vars = None
if model is not None:
try:
description = model.description
except:
description = "description missing"
description = dedent("""
{desc}
.. code:: python
{model}
{fvs}""").format(desc=description, model=model, fvs=forcing_vars)
else:
description = "Description missing - unknown model"
template = dedent("""
{title}
date: :code:`{date}`
Model details:
--------------
{desc}
Evaluation results:
-------------------
.. rst-class:: tablesorter
{eval_text}
Plots:
------
{plots}
""")
output = (template.format(model=model,
title=title,
desc=description,
plots=plots_text,
date=date,
eval_text=eval_text))
return output
def model_site_index_rst(name):
"""list site simulations for a model and print global model plots
:model_dir: Directory to create a model index in
"""
time = dt.isoformat(dt.now().replace(microsecond=0), sep=' ')
model_dir = 'source/models/' + name
try:
model = get_model(name)
try:
description = model.description
except:
description = "description missing"
description = dedent("""
{desc}
.. code:: python
`{model}`""").format(desc=description, model=model)
except:
description = "Description missing - unknown model"
logger.info('Generating index for {n}.'.format(n=name))
model_run_files = sorted(
glob.glob('{d}/{n}*.rst'.format(d=model_dir, n=name)))
sim_pages = [
os.path.splitext(os.path.basename(m))[0] for m in model_run_files
]
sim_links = '\n'.join([' {0}'.format(m) for m in sorted(sim_pages)])
try:
get_PLUMBER_plot(model_dir)
plot_files = glob.glob('{d}/figures/*.png'.format(d=model_dir))
rel_paths = ['figures/' + os.path.basename(p) for p in plot_files]
plots = '\n\n'.join([
".. image :: {file}\n :width: 300px".format(file=f)
for f in sorted(rel_paths)
])
except AttributeError as e:
logger.warning('No plots found, skipping {n}: {e}'.format(n=name, e=e))
return
title = '{name} simulations'.format(name=name)
title += '\n' + '=' * len(title)
template = dedent("""
{title}
{description}
{time}
Plots
-----
{plots}
Simulations
-----------
.. toctree::
:maxdepth: 1
{links}
""")
rst = template.format(time=time,
description=description,
links=sim_links,
title=title,
plots=plots)
rst_index = model_dir + '/index.rst'
with open(rst_index, 'w') as f:
f.write(rst)
return