def main_import_sim(name, site, sim_file): """import a PLUMBER benchmark for all sites :name: PLUMBER benchmark name :site: plumber site name """ # Hacky solution just for PLUMBER benchmarks print_good('Importing {n} data for: {s}'.format(n=name, s=site)) nc_path = get_sim_nc_path(name, site) data_vars = ['Qh', 'Qle', 'NEE'] with xr.open_dataset(sim_file) as ds: d_vars = [v for v in data_vars if v in ds] sim_data = ds[d_vars].copy(deep=True) if name == 'CHTESSEL': print("Inverting CHTESSEL and adding data") # F*****g inverted, and sim_data = -sim_data # missing most of the last f*****g day, and tsteps = ds.dims['time'] complete_tsteps = 48 * int(np.ceil(tsteps / 48)) missing_tsteps = complete_tsteps - tsteps new_data = (sim_data.isel(time=slice(-missing_tsteps, None)).copy( deep=True)) # f*****g off-set by an hour. with get_flux_data([site])[site] as ds: site_time = ds.time.values.flat.copy() sim_data['time'] = site_time[:tsteps] new_data['time'] = site_time[tsteps:] sim_data = xr.concat([sim_data, new_data], dim='time') if name == 'ORCHIDEE.trunk_r1401': sim_data.rename(dict(time_counter='time'), inplace=True) # Stores data for all vegetation types # NEE_veget_index = np.where(sim_data.NEE # .isel(time_counter=0, lat=0, lon=0) # .values.flat > 0)[0][0] # sim_data['NEE'] = sim_data['NEE'].isel(veget=NEE_veget_index) print("Flattening veg in ORCHIDEE") sim_data['NEE'] = sim_data['NEE'].sum(axis=1) # Sum over veget axis if site == 'Espirra': print("Deleting a year of data at Espirra for Orchidee") sim_data = sim_data.isel(time=slice(70128)) # WARNING! over writes existing sim! print('Writing to', nc_path) sim_data.to_netcdf(nc_path) sim_data.close() return
def main_import_benchmark(name, site): """import a PLUMBER benchmark for all sites :name: PLUMBER benchmark name :site: plumber site name """ # Hacky solution just for PLUMBER benchmarks print_good('Importing {n} data for: '.format(n=name)) if len(site) == 0: datasets = get_sites('PLUMBER') else: datasets = site for s in datasets: print(s, end=', ', flush=True) s_file = 'data/PALS/benchmarks/{n}/{n}_{s}Fluxnet.1.4.nc'.format( n=name, s=s) nc_path = get_sim_nc_path(name, s) sim_data = xr.open_dataset(s_file) fix_benchmark(sim_data, name, s) # WARNING! over writes existing sim! sim_data.to_netcdf(nc_path) sim_data.close() return
def eval_simulation(name, site, sim_file=None, plots=False, fix_closure=True, qc=True, overwrite=False): """Main function for evaluating an existing simulation. Copies simulation data to source directory. TODO: skip running if cached, for easier page regeneration :name: name of the model :site: PALS site name to run the model at :sim_file: Path to simulation netcdf. Only required if simulation is at a non-standard place. """ args = locals() args_str = '\n'.join([k + ': ' + str(args[k]) for k in sorted(args.keys())]) logger = setup_logger(__name__, 'logs/eval/{m}/{s}/{m}_{s}.log'.format(m=name, s=site)) logger.info("Evaluating model.\nArgs:\n{a}".format(a=args_str)) nc_path = get_sim_nc_path(name, site) if sim_file is None: filename = nc_path else: filename = sim_file eval_path = 'source/models/{n}/metrics/{n}_{s}_metrics.csv'.format(n=name, s=site) if not overwrite: if os.path.exists(filename) and os.path.exists(eval_path) and \ os.path.getmtime(filename) > os.path.getmtime(eval_path): logger.warning("Overwriting evaluation file because simulation is newer") else: logger.warning("Evaluation file already exists, skipping") return try: sim_data = xr.open_dataset(filename) except (OSError, RuntimeError) as e: logger.error("Sim file ({f}) doesn't exist. What are you doing? {e}".format(f=filename, e=e)) return if sim_file is not None: logger.warning("Overwriting existing sim!") sim_data.to_netcdf(nc_path) flux_data = get_flux_data([site], fix_closure=fix_closure)[site] evaluate_simulation(sim_data, flux_data, name, site=site, qc=qc) if plots: diagnostic_plots(sim_data, flux_data, name, site=site) sim_data.close() return
def eval_simulation(name, site, sim_file=None, plots=False, fix_closure=True, qc=True): """Main function for evaluating an existing simulation. Copies simulation data to source directory. TODO: skip running if cached, for easier page regeneration :name: name of the model :site: PALS site name to run the model at :sim_file: Path to simulation netcdf """ nc_path = get_sim_nc_path(name, site) if sim_file is None: filename = nc_path else: filename = sim_file try: sim_data = xr.open_dataset(filename) except (OSError, RuntimeError) as e: print_bad( "Sim file ({f}) doesn't exist. What are you doing? {e}".format( f=filename, e=e)) return if sim_file is not None: # WARNING! over writes existing sim! sim_data.to_netcdf(nc_path) flux_data = get_flux_data([site], fix_closure=fix_closure)[site] evaluate_simulation(sim_data, flux_data, name, qc=qc) if plots: diagnostic_plots(sim_data, flux_data, name) sim_data.close() return