def run_simulation_mp(name, site, no_mp=False, multivariate=False, overwrite=False, fix_closure=True): """Multi-processor run handling.""" # TODO: refactor to work with above caller. model = get_model(name) if site in ['all', 'PLUMBER_ext', 'PLUMBER']: logger.info('Running {n} at {s} sites'.format(n=name, s=site)) datasets = get_sites(site) if no_mp: for s in datasets: run_simulation(model, name, s, multivariate, overwrite, fix_closure) else: f_args = [(model, name, s, multivariate, overwrite, fix_closure) for s in datasets] ncores = get_suitable_ncores() if site is not 'debug' and hasattr(model, 'memory_requirement'): ncores = max( 1, int((psutil.virtual_memory().total / 2) // model.memory_requirement)) logger.info("Running on %d core(s)" % ncores) with Pool(ncores) as p: p.starmap(run_simulation, f_args) else: run_simulation(model, name, site, multivariate, overwrite, fix_closure) return
def fit_and_predict(name, dataset, years='2012-2013'): """Fit a benchmark to some PALS files, then generate an output matching a gridded dataset """ model = get_model(name) met_vars = model.forcing_vars flux_vars = ['Qle'] sites = get_sites('PLUMBER_ext') years = [int(s) for s in years.split('-')] print("Loading fluxnet data for %d sites" % len(sites)) met_data = pud.get_met_df(sites, met_vars, qc=True, name=True) flux_data = pud.get_flux_df(sites, flux_vars, qc=True) print("Fitting model {b} using {m} to predict {f}".format(n=name, m=met_vars, f=flux_vars)) model.fit(met_data, flux_data) # prediction datasets outdir = "{d}/gridded_benchmarks/{b}_{ds}".format(d=get_data_dir(), n=name, ds=dataset) os.makedirs(outdir, exist_ok=True) outfile_tpl = outdir + "/{b}_{d}_{v}_{y}.nc" for year in range(*years): print("Loading Forcing data for", year) data = get_dataset_data(dataset, met_vars, year) print("Predicting", year, end=': ', flush=True) if "lag" in name: result = predict_gridded(model, data, flux_vars, datafreq=get_dataset_freq(dataset)) else: result = predict_gridded(model, data, flux_vars) xr_add_attributes(result, model, dataset, sites) for fv in flux_vars: filename = outfile_tpl.format(n=name, d=dataset, v=fv, y=year) print("saving to ", filename) result[[fv]].to_netcdf(filename, encoding={fv: { 'dtype': 'float32' }}) return
def run_model_site_tuples_mp(tuples_list, no_mp=False, multivariate=False, overwrite=False, fix_closure=True): """Run (model, site) pairs""" seen = set() all_names = [ t[0] for t in tuples_list if not (t[0] in seen or seen.add(t[0])) ] all_models = {n: get_model(n) for n in all_names} f_args = [(all_models[t[0]], t[0], t[1], multivariate, overwrite, fix_closure) for t in tuples_list] if no_mp: [run_simulation(*args) for args in f_args] else: # multiprocess ncores = get_suitable_ncores() # TODO: Deal with memory requirement? with Pool(ncores) as p: p.starmap(run_simulation, f_args)
def model_site_rst_format(name, site, eval_text, plot_files): """format all the datas into an rst! """ date = dt.isoformat(dt.now().replace(microsecond=0), sep=' ') plots_text = '' for group in sorted(plot_files): plots_text += "{g}\n".format(g=group) plots_text += "^" * len(group) + "\n\n" plots_text += '\n\n'.join([ ".. image :: {file}\n :width: 200px".format(file=f) for f in sorted(plot_files[group]) ]) plots_text += '\n\n' title = '{name} at {site}'.format(name=name, site=site) title += '\n' + '=' * len(title) try: model = get_model(name) except: model = None if hasattr(model, "_var_lags"): forcing_vars = model._var_lags elif hasattr(model, "forcing_vars"): forcing_vars = model.forcing_vars else: forcing_vars = None if model is not None: try: description = model.description except: description = "description missing" description = dedent(""" {desc} .. code:: python {model} {fvs}""").format(desc=description, model=model, fvs=forcing_vars) else: description = "Description missing - unknown model" template = dedent(""" {title} date: :code:`{date}` Model details: -------------- {desc} Evaluation results: ------------------- .. rst-class:: tablesorter {eval_text} Plots: ------ {plots} """) output = (template.format(model=model, title=title, desc=description, plots=plots_text, date=date, eval_text=eval_text)) return output
def model_site_index_rst(name): """list site simulations for a model and print global model plots :model_dir: Directory to create a model index in """ time = dt.isoformat(dt.now().replace(microsecond=0), sep=' ') model_dir = 'source/models/' + name try: model = get_model(name) try: description = model.description except: description = "description missing" description = dedent(""" {desc} .. code:: python `{model}`""").format(desc=description, model=model) except: description = "Description missing - unknown model" logger.info('Generating index for {n}.'.format(n=name)) model_run_files = sorted( glob.glob('{d}/{n}*.rst'.format(d=model_dir, n=name))) sim_pages = [ os.path.splitext(os.path.basename(m))[0] for m in model_run_files ] sim_links = '\n'.join([' {0}'.format(m) for m in sorted(sim_pages)]) try: get_PLUMBER_plot(model_dir) plot_files = glob.glob('{d}/figures/*.png'.format(d=model_dir)) rel_paths = ['figures/' + os.path.basename(p) for p in plot_files] plots = '\n\n'.join([ ".. image :: {file}\n :width: 300px".format(file=f) for f in sorted(rel_paths) ]) except AttributeError as e: logger.warning('No plots found, skipping {n}: {e}'.format(n=name, e=e)) return title = '{name} simulations'.format(name=name) title += '\n' + '=' * len(title) template = dedent(""" {title} {description} {time} Plots ----- {plots} Simulations ----------- .. toctree:: :maxdepth: 1 {links} """) rst = template.format(time=time, description=description, links=sim_links, title=title, plots=plots) rst_index = model_dir + '/index.rst' with open(rst_index, 'w') as f: f.write(rst) return