import xarray as xr import sys import os import pandas as pd from tonic.io import read_configobj cfg = read_configobj(sys.argv[1]) i = int(sys.argv[2]) def to_netcdf_history_file_compress(ds_hist, out_nc): ''' This function saves a VIC-history-file-format ds to netCDF, with compression. Parameters ---------- ds_hist: <xr.Dataset> History dataset to save out_nc: <str> Path of output netCDF file ''' dict_encode = {} for var in ds_hist.data_vars: # skip variables not starting with "OUT_" if var.split('_')[0] != 'OUT': continue # determine chunksizes chunksizes = [] for i, dim in enumerate(ds_hist[var].dims):
import os import xarray as xr import subprocess import string import pandas as pd import warnings warnings.filterwarnings('ignore') from optimize_utils import read_RVIC_output, read_USGS_data, kge from tonic.io import read_configobj # ======================================================== # # Parse in parameters from MOCOM # ======================================================== # cfg = read_configobj(sys.argv[1]) infilt = float(sys.argv[2]) d1 = float(sys.argv[3]) d2 = float(sys.argv[4]) d3 = float(sys.argv[5]) expt = float(sys.argv[6]) Ksat = float(sys.argv[7]) statsfile = sys.argv[8] stor_dir = sys.argv[9] # ======================================================== # # Parameter setting # ======================================================== # # --- VIC --- #
def main(cfg_file, nproc=1): ''' Main function Parameters ---------- cfg_file: <str> Input config file nproc: <int> Number of processors to use ''' # ====================================================== # # Load in config file # ====================================================== # cfg = read_configobj(cfg_file) # ====================================================== # # Process some cfg variables # ====================================================== # start_date = pd.to_datetime(cfg['FORCING']['start_date']) end_date = pd.to_datetime(cfg['FORCING']['end_date']) start_year = start_date.year end_year = end_date.year ens_list = range(cfg['FORCING']['ens_start'], cfg['FORCING']['ens_end'] + 1) # ====================================================== # # Set up output directories # ====================================================== # dirs = setup_output_dirs(cfg['OUTPUT']['out_basedir'], mkdirs=['forc_orig_nc', 'forc_orig_asc', 'forc_disagg_asc', 'forc_disagg_nc', 'config_files', 'logs_vic']) # Subdirs for config files for ensemble subdirs_config = setup_output_dirs( dirs['config_files'], mkdirs=['netcdf2vic', 'vic4', 'vic2nc']) # ====================================================== # # Load in domain file # ====================================================== # ds_domain = xr.open_dataset(cfg['DOMAIN']['domain_nc']) da_domain = ds_domain[cfg['DOMAIN']['mask_name']] lat_min = da_domain['lat'].min().values lat_max = da_domain['lat'].max().values lon_min = da_domain['lon'].min().values lon_max = da_domain['lon'].max().values # ====================================================== # # Load in and process Newman ensemble forcings (for prec, Tmax and Tmin) # and orig. Maurer forcing (for wind speed) # ====================================================== # # --- Load Maurer forcings --- # print('Processing Maurer forcings...') # Loop over each year list_da_wind = [] for year in range(start_year, end_year+1): print('Year {}'.format(year)) # --- Load in netCDF file for this year --- # da_wind = xr.open_dataset(os.path.join( cfg['FORCING']['maurer_dir'], 'nldas_met_update.obs.daily.wind.{}.nc'.format(year)))['wind'] # --- Mask out the target area --- # da_wind = da_wind.sel(latitude=slice(lat_min, lat_max), longitude=slice(lon_min, lon_max)) da_wind = da_wind.where(da_domain.values) # --- Rename lat and lon --- # da_wind = da_wind.rename({'latitude': 'lat', 'longitude': 'lon'}) # --- Put in list --- # list_da_wind.append(da_wind) # Concat all years together da_wind_allyears = xr.concat(list_da_wind, dim='time') # --- Load Newman forcings --- # print('Processing Newman forcings...') # If 1 processor, do a regular process if nproc == 1: # Loop over each ensemble member for ens in ens_list: load_and_process_Newman(ens, cfg, da_domain, lat_min, lat_max, lon_min, lon_max, start_date, end_date, dirs, da_wind_allyears) # If multiple processors, use mp elif nproc > 1: # Set up multiprocessing pool = mp.Pool(processes=nproc) # Loop over each ensemble member for ens in ens_list: pool.apply_async(load_and_process_Newman, (ens, cfg, da_domain, lat_min, lat_max, lon_min, lon_max, start_date, end_date, dirs, da_wind_allyears,)) # Finish multiprocessing pool.close() pool.join() # ====================================================== # # Convert orig. forcings to ascii format # ====================================================== # print('Converting orig. netCDF forcings to VIC ascii...') # --- Setup subdirs for asc VIC orig. forcings for each ensemble member # --- # list_ens = [] for ens in ens_list: list_ens.append('ens_{}'.format(ens)) subdirs_output = setup_output_dirs( dirs['forc_orig_asc'], mkdirs=list_ens) # --- Prepare netcdf2vic config file --- # dict_cfg_file = {} for ens in ens_list: cfg_file = os.path.join(subdirs_config['netcdf2vic'], 'ens_{}.cfg'.format(ens)) dict_cfg_file[ens] = cfg_file with open(cfg_file, 'w') as f: f.write('[options]\n') f.write('files: forc_orig.{}.nc\n') f.write('verbose: True\n') f.write('output_format: ASCII\n') f.write('out_prefix: forc_orig_\n') f.write('coord_keys: lon,lat\n') f.write('var_keys: pr,tasmax,tasmin,wind\n') f.write('start_year: {}\n'.format(start_year)) f.write('end_year: {}\n'.format(end_year)) f.write('latlon_precision: {}\n'.format( cfg['OUTPUT']['latlon_precision'])) f.write('\n[paths]\n') f.write('in_path: {}\n'.format(os.path.join( dirs['forc_orig_nc'], 'ens_{}'.format(ens)))) f.write('mask_path: {}\n'.format(cfg['DOMAIN']['domain_nc'])) f.write('mask_varname: {}\n'.format(cfg['DOMAIN']['mask_name'])) f.write('ASCIIoutPath: {}\n'.format( subdirs_output['ens_{}'.format(ens)])) # --- Run nc_to_vic --- # # If 1 processor, do a regular process if nproc == 1: for ens in ens_list: nc_to_vic(dict_cfg_file[ens]) # If multiple processors, use mp elif nproc > 1: # Set up multiprocessing pool = mp.Pool(processes=nproc) # Loop over each ensemble member for ens in ens_list: pool.apply_async(nc_to_vic, (dict_cfg_file[ens],)) # Finish multiprocessing pool.close() pool.join() # ====================================================== # # Run VIC forcing disaggregator # ====================================================== # print('Running VIC as a disaggregator...') # --- Setup subdirs for asc VIC disagg. forcings and VIC log files for # each ensemble member --- # list_ens = [] for ens in ens_list: list_ens.append('ens_{}'.format(ens)) subdirs_output = setup_output_dirs( dirs['forc_disagg_asc'], mkdirs=list_ens) subdirs_logs = setup_output_dirs( dirs['logs_vic'], mkdirs=list_ens) # --- Prepare VIC global file for the disaggregation run --- # # Load in global file template with open(cfg['VIC_DISAGG']['global_template'], 'r') as f: global_param = f.read() # Create string template s = string.Template(global_param) # Loop over each ensemble member dict_global_file = {} for ens in ens_list: # Fill in variables in the template global_param = s.safe_substitute( time_step=cfg['VIC_DISAGG']['time_step'], startyear=start_year, startmonth=start_date.month, startday=start_date.day, endyear=end_year, endmonth=end_date.month, endday=end_date.day, forcing1=os.path.join(dirs['forc_orig_asc'], 'ens_{}'.format(ens), 'forc_orig_'), grid_decimal=cfg['OUTPUT']['latlon_precision'], prec='PREC', tmax='TMAX', tmin='TMIN', wind='WIND', forceyear=start_year, forcemonth=start_date.month, forceday=start_date.day, result_dir=subdirs_output['ens_{}'.format(ens)]) # Write global param file global_file = os.path.join(subdirs_config['vic4'], 'vic.global.ens_{}.txt'.format(ens)) dict_global_file[ens] = global_file with open(global_file, mode='w') as f: for line in global_param: f.write(line) # --- Run VIC --- # # Prepare VIC exe vic_exe = VIC(cfg['VIC_DISAGG']['vic4_exe']) # If 1 processor, do a regular process if nproc == 1: for ens in ens_list: vic_exe.run(dict_global_file[ens], logdir=subdirs_logs['ens_{}'.format(ens)]) # If multiple processors, use mp elif nproc > 1: # Set up multiprocessing pool = mp.Pool(processes=nproc) # Loop over each ensemble member for ens in ens_list: pool.apply_async(run_vic_for_multiprocess, (vic_exe, dict_global_file[ens], subdirs_logs['ens_{}'.format(ens)],)) # Finish multiprocessing pool.close() pool.join() # ====================================================== # # Convert disaggregated forcings to netCDF format # ====================================================== # # --- Prepare config file for vic2nc --- # print('Converting disaggregated forcings to netCDF...') # --- Setup subdirs for VIC disagg. netCDF forcings for each ensemble # member --- # list_ens = [] for ens in ens_list: list_ens.append('ens_{}'.format(ens)) subdirs_output = setup_output_dirs( dirs['forc_disagg_nc'], mkdirs=list_ens) # --- Prepare netcdf2vic config file --- # # Extract disaggregated forcing variable names and order with open(cfg['VIC_DISAGG']['global_template'], 'r') as f: global_param = f.read() outvar_list = find_outvar_global_param(global_param) for i, var in enumerate(outvar_list): outvar_list[i] = var.strip('OUT_') # Extract end date and hour end_date_with_hour = end_date + pd.DateOffset(days=1) -\ pd.DateOffset(hours=cfg['VIC_DISAGG']['time_step']) # Loop over each ensemble member dict_cfg_file = {} for ens in ens_list: cfg_file = os.path.join(subdirs_config['vic2nc'], 'ens_{}.cfg'.format(ens)) dict_cfg_file[ens] = cfg_file with open(cfg_file, 'w') as f: f.write('[OPTIONS]\n') f.write('input_files: {}\n'.format( os.path.join(dirs['forc_disagg_asc'], 'ens_{}'.format(ens), 'force_*'))) f.write('input_file_format: ascii\n') f.write('bin_dt_sec: {}\n'.format(cfg['VIC_DISAGG']['time_step']*3600)) f.write('bin_start_date: {}\n'.format(start_date.strftime("%Y-%m-%d-%H"))) f.write('bin_end_date: {}\n'.format(end_date_with_hour.strftime("%Y-%m-%d-%H"))) f.write('regular_grid: False\n') f.write('out_directory: {}\n'.format(subdirs_output['ens_{}'.format(ens)])) f.write('memory_mode: big_memory\n') f.write('chunksize: 100\n') f.write('out_file_prefix: force\n') f.write('out_file_format: NETCDF4\n') f.write('precision: single\n') f.write('start_date: {}\n'.format(start_date.strftime("%Y-%m-%d-%H"))) f.write('end_date: {}\n'.format(end_date_with_hour.strftime("%Y-%m-%d-%H"))) f.write('calendar: proleptic_gregorian\n') f.write('time_segment: year\n') f.write('snow_bands: False\n') f.write('veg_tiles: False\n') f.write('soil_layers: False\n') f.write('\n[DOMAIN]\n') f.write('filename: {}\n'.format(cfg['DOMAIN']['domain_nc'])) f.write('longitude_var: {}\n'.format(cfg['DOMAIN']['lon_name'])) f.write('latitude_var: {}\n'.format(cfg['DOMAIN']['lat_name'])) f.write('y_x_dims: {}, {}\n'.format(cfg['DOMAIN']['lat_name'], cfg['DOMAIN']['lon_name'])) f.write('copy_vars: {}, {}, {}\n'.format(cfg['DOMAIN']['mask_name'], cfg['DOMAIN']['lat_name'], cfg['DOMAIN']['lon_name'])) f.write('\n[GLOBAL_ATTRIBUTES]\n') f.write('title: VIC forcings\n') f.write('version: VIC4.2\n') f.write('grid: 1/8\n') for i, var in enumerate(outvar_list): if var == 'AIR_TEMP': f.write('\n[AIR_TEMP]\n') f.write('column: {}\n'.format(i)) f.write('units: C\n') f.write('standard_name: air_temperature\n') f.write('description: air temperature\n') elif var == 'PREC': f.write('\n[PREC]\n') f.write('column: {}\n'.format(i)) f.write('units: mm/step\n') f.write('standard_name: precipitation\n') f.write('description: precipitation\n') elif var == 'PRESSURE': f.write('\n[PRESSURE]\n') f.write('column: {}\n'.format(i)) f.write('units: kPa\n') f.write('standard_name: surface_air_pressure\n') f.write('description: near-surface atmospheric pressure\n') elif var == 'SHORTWAVE': f.write('\n[SHORTWAVE]\n') f.write('column: {}\n'.format(i)) f.write('units: W m-2\n') f.write('standard_name: incoming_shortwave_radiation\n') f.write('description: incoming shortwave radiation\n') elif var == 'LONGWAVE': f.write('\n[LONGWAVE]\n') f.write('column: {}\n'.format(i)) f.write('units: W m-2\n') f.write('standard_name: incoming_longwave_radiation\n') f.write('description: incoming longwave radiation\n') elif var == 'VP': f.write('\n[VP]\n') f.write('column: {}\n'.format(i)) f.write('units: kPa\n') f.write('standard_name: water_vapor_pressure\n') f.write('description: near surface vapor pressure\n') elif var == 'WIND': f.write('\n[WIND]\n') f.write('column: {}\n'.format(i)) f.write('units: m/s\n') f.write('standard_name: surface_air_pressure\n') f.write('description: near-surface wind speed\n') # --- Run vic2nc --- # # If 1 processor, do a regular process if nproc == 1: for ens in ens_list: cfg_vic2nc = read_config(dict_cfg_file[ens]) options = cfg_vic2nc.pop('OPTIONS') global_atts = cfg_vic2nc.pop('GLOBAL_ATTRIBUTES') if not options['regular_grid']: domain_dict = cfg_vic2nc.pop('DOMAIN') else: domain_dict = None # Set aside fields dict fields = cfg_vic2nc # Run vic2nc vic2nc(options, global_atts, domain_dict, fields) # If multiple processors, use mp elif nproc > 1: # Set up multiprocessing pool = mp.Pool(processes=nproc) # Loop over each ensemble member for ens in ens_list: cfg_vic2nc = read_config(dict_cfg_file[ens]) options = cfg_vic2nc.pop('OPTIONS') global_atts = cfg_vic2nc.pop('GLOBAL_ATTRIBUTES') if not options['regular_grid']: domain_dict = cfg_vic2nc.pop('DOMAIN') else: domain_dict = None # set aside fields dict fields = cfg_vic2nc pool.apply_async(vic2nc, (options, global_atts, domain_dict, fields,)) # Finish multiprocessing pool.close() pool.join()
def run_system(config_file, vic_exe, test_data_dir, out_dir, driver): '''Run system tests from config file Parameters ---------- config_file : str Configuration file for system tests. vic_exe : VIC (object) VIC executable object (see tonic documentation). test_data_dir : str Path to test data sets. out_dir : str Path to output location driver : {'classic', 'image'} Driver to run tests on. Returns ------- test_results : dict Test results for all tests in config_file. See Also -------- run_unit_tests run_examples run_science run_release ''' # Print test set welcome print('\n-'.ljust(OUTPUT_WIDTH + 1, '-')) print('Running System Tests') print('-'.ljust(OUTPUT_WIDTH, '-')) # Get setup config = read_configobj(config_file) # drop invalid driver tests config = drop_tests(config, driver) test_results = OrderedDict() # Run individual system tests for i, (testname, test_dict) in enumerate(config.items()): # print out status info print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()), testname)) # Setup directories for test dirs = setup_test_dirs(testname, out_dir, mkdirs=['results', 'state', 'logs', 'plots']) # read template global parameter file infile = os.path.join(test_dir, 'system', test_dict['global_parameter_file']) with open(infile, 'r') as global_file: global_param = global_file.read() # If restart test, prepare running periods # (1) Find STATESEC option (and STATE_FORMAT option for later use) statesec = find_global_param_value(global_param, 'STATESEC') if driver == 'classic': state_format = find_global_param_value(global_param, 'STATE_FORMAT') # (2) Prepare running periods and initial state file info for restart # test if 'exact_restart' in test_dict['check']: run_periods = prepare_restart_run_periods( test_dict['restart'], dirs['state'], statesec) # create template string s = string.Template(global_param) # fill in global parameter options # --- if restart test, multiple runs --- # if 'exact_restart' in test_dict['check']: # Set up subdirectories and fill in global parameter options # for restart testing list_global_param =\ setup_subdirs_and_fill_in_global_param_restart_test( s, run_periods, driver, dirs['results'], dirs['state'], test_data_dir) # else, single run else: global_param = s.safe_substitute(test_data_dir=test_data_dir, result_dir=dirs['results'], state_dir=dirs['state']) # replace global options from config file # --- extract global options to be substitute --- # if 'options' in test_dict: replacements = test_dict['options'] else: replacements = OrderedDict() # --- if STATE_FORMAT is specified, then the specified value (instead # of the one in the global template file) --- # if 'STATE_FORMAT' in replacements: state_format = replacements['STATE_FORMAT'] # --- replace global options --- # if 'exact_restart' in test_dict['check']: for j, gp in enumerate(list_global_param): # save a copy of replacements for the next global file replacements_cp = replacements.copy() # replace global options for this global file list_global_param[j] = replace_global_values(gp, replacements) replacements = replacements_cp else: global_param = replace_global_values(global_param, replacements) # write global parameter file if 'exact_restart' in test_dict['check']: list_test_global_file = [] for j, gp in enumerate(list_global_param): test_global_file = os.path.join( dirs['test'], '{}_globalparam_{}_{}.txt'.format( testname, run_periods[j]['start_date'].strftime("%Y%m%d"), run_periods[j]['end_date'].strftime("%Y%m%d"))) list_test_global_file.append(test_global_file) with open(test_global_file, mode='w') as f: for line in gp: f.write(line) else: test_global_file = os.path.join( dirs['test'], '{0}_globalparam.txt'.format(testname)) with open(test_global_file, mode='w') as f: for line in global_param: f.write(line) # Get optional kwargs for run executable run_kwargs = pop_run_kwargs(test_dict) # run VIC test_complete = False test_passed = False test_comment = '' error_message = '' try: if 'exact_restart' in test_dict['check']: for j, test_global_file in enumerate(list_test_global_file): returncode = vic_exe.run(test_global_file, logdir=dirs['logs']) # Check return code check_returncode(vic_exe, test_dict.pop('expected_retval', 0)) else: returncode = vic_exe.run(test_global_file, logdir=dirs['logs'], **run_kwargs) # Check return code check_returncode(vic_exe, test_dict.pop('expected_retval', 0)) test_complete = True # check output files (different tests depending on driver) if 'check' in test_dict: # Check that the simulation completed for all grid cells if 'complete' in test_dict['check']: fnames = glob.glob(os.path.join(dirs['results'], '*')) if driver == 'classic': test_classic_driver_all_complete(fnames) # check for nans in all example files if 'output_file_nans' in test_dict['check']: fnames = glob.glob(os.path.join(dirs['results'], '*')) if driver == 'classic': test_classic_driver_no_output_file_nans(fnames) elif driver == 'image': domain_file = os.path.join(test_data_dir, test_dict['domain_file']) test_image_driver_no_output_file_nans(fnames, domain_file) else: raise ValueError('unknown driver') # check for exact restarts if 'exact_restart' in test_dict['check']: check_exact_restart_fluxes(dirs['results'], driver, run_periods) if driver == 'classic': check_exact_restart_states(dirs['state'], driver, run_periods, statesec, state_format) elif driver == 'image': check_exact_restart_states(dirs['state'], driver, run_periods, statesec) if 'multistream' in test_dict['check']: fnames = glob.glob(os.path.join(dirs['results'], '*')) if driver == 'classic': check_multistream_classic(fnames) elif driver == 'image': warnings.warn('Skipping multistream image driver test') # TODO: check_multistream_image(fnames) # if we got this far, the test passed. test_passed = True # Handle errors except Exception as e: test_comment, error_message = process_error(e, vic_exe) # record the test results test_results[testname] = TestResults(testname, test_complete=test_complete, passed=test_passed, comment=test_comment, error_message=error_message, returncode=returncode) print('-'.ljust(OUTPUT_WIDTH, '-')) print('Finished testing system tests.') print('-'.ljust(OUTPUT_WIDTH, '-')) return test_results
#!/usr/local/anaconda/bin/python ''' This scripts prepares a xmask file for Lohmann routine, using consistant method as in the inverse routing method (i.e., using haversine formula) ''' import numpy as np import argparse from tonic.io import read_configobj from da_utils import generate_xmask_for_route parser = argparse.ArgumentParser() parser.add_argument("--cfg", type=str, help="config file for this script") args = parser.parse_args() cfg = read_configobj(args.cfg) #========================================================# # Calculate xmask (i.e., flow distance) #========================================================# flow_distance = generate_xmask_for_route(cfg['INPUT']['fdir_path']) #========================================================# # Write to file #========================================================# f = open(cfg['OUTPUT']['output_xmask_path'], 'w') #--- Write header lines (copy from flow direction file) ---# f_fdir = open(cfg['INPUT']['fdir_path'], 'r') for i in range(6): line = f_fdir.readline().rstrip("\n") # read from flow direction file f.write(line + "\n") f_fdir.close() #--- Write xmask values ---#
matplotlib.use('Agg') import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec from sklearn.neighbors import NearestNeighbors from tonic.io import read_config, read_configobj from utils import calc_monthly_PET, find_1deg_grid_gldas, edges_from_centers # ========================================= # # Parameter setting # ========================================= # # Load both SMAP and GLDAS, v1 and v2 results dict_cfg = {} for v in ['v1', 'v2']: dict_cfg['SMAP_{}'.format(v)] = read_configobj( '/civil/hydro/ymao/smap_data/scripts/cfg/20181121.SMAP_recom_qual.exclude_arid/' 'X_{}.linear.cfg'.format(v)) dict_cfg['GLDAS_{}'.format(v)] = read_configobj( '/civil/hydro/ymao/smap_data/scripts/cfg/20181121.GLDAS_VIC.exclude_arid/smap_freq/' 'X_{}.linear.cfg'.format(v)) # Output data dir output_dir = 'output/20181121.smap_gldas' # --- Load GLDAS monthly data --- # ds_monthly = xr.open_dataset( '/civil/hydro/ymao/smap_data/data/GLDAS/VIC_monthly/aggregated_variables/soil_moisture.197901_201712.monthly.nc' ) ds_monthly.load() ds_monthly.close()
def run_system(config_file, dict_drivers, test_data_dir, out_dir): '''Run system tests from config file Parameters ---------- config_file : str Configuration file for system tests. dict_drivers : dict Keys: driver names {'classic', 'image'} Content: corresponding VIC executable object (see tonic documentation) test_data_dir : str Path to test data sets. out_dir : str Path to output location Returns ------- test_results : dict Test results for all tests in config_file. See Also -------- run_unit_tests run_examples run_science run_release ''' # Print test set welcome print('\n-'.ljust(OUTPUT_WIDTH + 1, '-')) print('Running System Tests') print('-'.ljust(OUTPUT_WIDTH, '-')) # Get setup config = read_configobj(config_file) # Process driver info if len(dict_drivers) == 1: # if single driver driver = list(dict_drivers.keys())[0] vic_exe = dict_drivers[driver] # Drop invalid driver tests if len(dict_drivers) == 1: # if single driver config = drop_tests(config, driver) else: # if multiple drivers config = drop_tests(config, list(dict_drivers)) test_results = OrderedDict() # Run individual system tests for i, (testname, test_dict) in enumerate(config.items()): # print out status info print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()), testname)) # Setup directories for test dirs = setup_test_dirs(testname, out_dir, mkdirs=['results', 'state', 'logs', 'plots']) # read template global parameter file dict_global_param = {} # --- if single driver --- # if len(dict_drivers) == 1: infile = os.path.join(test_dir, 'system', test_dict['global_parameter_file']) with open(infile, 'r') as global_file: dict_global_param[driver] = global_file.read() # --- if multiple drivers --- # else: for j, dr in enumerate(test_dict['driver']): infile = os.path.join(test_dir, 'system', test_dict['global_parameter_file'][j]) with open(infile, 'r') as global_file: dict_global_param[dr] = global_file.read() # If restart test, prepare running periods if 'exact_restart' in test_dict['check']: if len(dict_drivers) > 1: raise ValueError('Only support single driver for restart' 'tests!') global_param = dict_global_param[driver] # (1) Find STATESEC option (and STATE_FORMAT option for later use) statesec = find_global_param_value(global_param, 'STATESEC') if driver == 'classic': state_format = find_global_param_value(global_param, 'STATE_FORMAT') # (2) Prepare running periods and initial state file info for # restart test run_periods = prepare_restart_run_periods(test_dict['restart'], dirs['state'], statesec) # If mpi test, prepare a list of number of processors to be run elif 'mpi' in test_dict['check']: if len(dict_drivers) > 1: raise ValueError('Only support single driver for MPI' 'tests!') if not isinstance(test_dict['mpi']['n_proc'], list): raise ValueError('Need at least two values in n_proc to run' 'mpi test!') list_n_proc = test_dict['mpi']['n_proc'] # create template string dict_s = {} for dr, global_param in dict_global_param.items(): dict_s[dr] = string.Template(global_param) # fill in global parameter options # --- if restart test, multiple runs --- # if 'exact_restart' in test_dict['check']: s = dict_s[driver] # Set up subdirectories and fill in global parameter options # for restart testing list_global_param =\ setup_subdirs_and_fill_in_global_param_restart_test( s, run_periods, driver, dirs['results'], dirs['state'], test_data_dir) # --- if mpi test, multiple runs --- # elif 'mpi' in test_dict['check']: s = dict_s[driver] # Set up subdirectories and output directories in global file for # multiprocessor testing list_global_param = \ setup_subdirs_and_fill_in_global_param_mpi_test( s, list_n_proc, dirs['results'], dirs['state'], test_data_dir) # --- if driver-match test, one run for each driver --- # elif 'driver_match' in test_dict['check']: # Set up subdirectories and output directories in global file for # driver-match testing dict_global_param = \ setup_subdirs_and_fill_in_global_param_driver_match_test( dict_s, dirs['results'], dirs['state'], test_data_dir) # --- else, single run --- # else: if len(dict_drivers) > 1: raise RuntimeError('Only support single driver for test' '{}!'.format(testname)) s = dict_s[driver] global_param = s.safe_substitute(test_data_dir=test_data_dir, result_dir=dirs['results'], state_dir=dirs['state']) # replace global options from config file # --- extract global options to be substitute --- # if 'options' in test_dict: replacements = test_dict['options'] else: replacements = OrderedDict() # --- replace global options --- # # For the purpose of exact restart, if STATE_FORMAT is specified, # then record the specified value (instead of the one in the global # template file) if 'exact_restart' in test_dict['check']: if 'STATE_FORMAT' in replacements: state_format = replacements['STATE_FORMAT'] if 'exact_restart' in test_dict['check'] or\ 'mpi' in test_dict['check']: # if multiple runs for j, gp in enumerate(list_global_param): # save a copy of replacements for the next global file replacements_cp = replacements.copy() # replace global options for this global file list_global_param[j] = replace_global_values(gp, replacements) replacements = replacements_cp elif 'driver_match' in test_dict['check']: # if cross-driver runs for dr, gp in dict_global_param.items(): # save a copy of replacements for the next global file replacements_cp = replacements.copy() # replace global options for this global file dict_global_param[dr] = replace_global_values(gp, replacements) replacements = replacements_cp else: # if single run global_param = replace_global_values(global_param, replacements) # write global parameter file if 'exact_restart' in test_dict['check']: list_test_global_file = [] for j, gp in enumerate(list_global_param): test_global_file = os.path.join( dirs['test'], '{}_globalparam_{}_{}.txt'.format( testname, run_periods[j]['start_date'].strftime("%Y%m%d"), run_periods[j]['end_date'].strftime("%Y%m%d"))) list_test_global_file.append(test_global_file) with open(test_global_file, mode='w') as f: for line in gp: f.write(line) elif 'mpi' in test_dict['check']: list_test_global_file = [] for j, gp in enumerate(list_global_param): test_global_file = os.path.join( dirs['test'], '{}_globalparam_processors_{}.txt'.format( testname, list_n_proc[j])) list_test_global_file.append(test_global_file) with open(test_global_file, mode='w') as f: for line in gp: f.write(line) elif 'driver_match' in test_dict['check']: dict_test_global_file = {} for dr, gp in dict_global_param.items(): test_global_file = os.path.join( dirs['test'], '{}_globalparam_{}.txt'.format(testname, dr)) dict_test_global_file[dr] = test_global_file with open(test_global_file, mode='w') as f: for line in gp: f.write(line) else: test_global_file = os.path.join( dirs['test'], '{0}_globalparam.txt'.format(testname)) with open(test_global_file, mode='w') as f: for line in global_param: f.write(line) # Get optional kwargs for run executable run_kwargs = pop_run_kwargs(test_dict) # run VIC test_complete = False test_passed = False test_comment = '' error_message = '' try: if 'exact_restart' in test_dict['check']: for j, test_global_file in enumerate(list_test_global_file): returncode = vic_exe.run(test_global_file, logdir=dirs['logs'], **run_kwargs) # Check return code check_returncode(vic_exe, test_dict.pop('expected_retval', 0)) elif 'mpi' in test_dict['check']: for j, test_global_file in enumerate(list_test_global_file): # Overwrite mpi_proc in option kwargs n_proc = list_n_proc[j] if n_proc == 1: run_kwargs['mpi_proc'] = None else: run_kwargs['mpi_proc'] = list_n_proc[j] # Run VIC returncode = vic_exe.run(test_global_file, logdir=dirs['logs'], **run_kwargs) # Check return code check_returncode(vic_exe, test_dict.pop('expected_retval', 0)) elif 'driver_match' in test_dict['check']: for dr in dict_test_global_file.keys(): # Reset mpi_proc in option kwargs to None for classic # driver run if dr == 'classic': run_kwargs_classic = run_kwargs run_kwargs_classic['mpi_proc'] = None returncode = dict_drivers[dr].run( dict_test_global_file[dr], logdir=dirs['logs'], **run_kwargs_classic) else: returncode = dict_drivers[dr].run( dict_test_global_file[dr], logdir=dirs['logs'], **run_kwargs) # Check return code check_returncode(dict_drivers[dr], test_dict.pop('expected_retval', 0)) else: returncode = vic_exe.run(test_global_file, logdir=dirs['logs'], **run_kwargs) # Check return code check_returncode(vic_exe, test_dict.pop('expected_retval', 0)) test_complete = True # check output files (different tests depending on driver) if 'check' in test_dict: # Check that the simulation completed for all grid cells if 'complete' in test_dict['check']: if len(dict_drivers) > 1: raise RuntimeError('Only support single driver for ' 'complete check') fnames = glob.glob(os.path.join(dirs['results'], '*')) if driver == 'classic': test_classic_driver_all_complete(fnames) else: raise RuntimeError('complete check only supports ' 'classic driver') # check for nans in all example files if 'output_file_nans' in test_dict['check']: if len(dict_drivers) > 1: raise RuntimeError('Only support single driver for ' 'output_file_nans check') fnames = glob.glob(os.path.join(dirs['results'], '*')) if driver == 'classic': test_classic_driver_no_output_file_nans(fnames) elif driver == 'image': domain_file = os.path.join(test_data_dir, test_dict['domain_file']) test_image_driver_no_output_file_nans( fnames, domain_file) else: raise ValueError('unknown driver') # check for exact restarts if 'exact_restart' in test_dict['check']: check_exact_restart_fluxes(dirs['results'], driver, run_periods) if driver == 'classic': check_exact_restart_states(dirs['state'], driver, run_periods, statesec, state_format) elif driver == 'image': check_exact_restart_states(dirs['state'], driver, run_periods, statesec) else: raise ValueError('unknown driver') # check for multistream output if 'multistream' in test_dict['check']: if len(dict_drivers) > 1: raise ValueError('Only support single driver for ' 'multistream check') fnames = glob.glob(os.path.join(dirs['results'], '*')) if driver == 'classic': check_multistream_classic(fnames) elif driver == 'image': warnings.warn('Skipping multistream image driver test') # TODO: check_multistream_image(fnames) # check for mpi multiprocessor results if 'mpi' in test_dict['check']: check_mpi_fluxes(dirs['results'], list_n_proc) check_mpi_states(dirs['state'], list_n_proc) # check that results from different drivers match if 'driver_match' in test_dict['check']: check_drivers_match_fluxes(list(dict_drivers.keys()), dirs['results']) # if we got this far, the test passed. test_passed = True # Handle errors except Exception as e: for dr, exe in dict_drivers.items(): test_comment, error_message = process_error(e, exe) # record the test results test_results[testname] = TestResults(testname, test_complete=test_complete, passed=test_passed, comment=test_comment, error_message=error_message, returncode=returncode) print('-'.ljust(OUTPUT_WIDTH, '-')) print('Finished testing system tests.') print('-'.ljust(OUTPUT_WIDTH, '-')) return test_results
def run_science(config_file, vic_exe, science_test_data_dir, test_data_dir, out_dir, driver, nproc): '''Run science tests from config file Parameters ---------- config_file : str Configuration file for science tests. vic_exe : VIC (object) VIC executable object (see tonic documentation). science_test_data_dir: str Path to science test data sets (archived VIC runs and observations) test_data_dir : str Path to test data sets. out_dir : str Path to output location driver : {'classic', 'image'} Driver to run tests on. nproc : int Number of processors to use for science tests Returns ------- test_results : dict Test results for all tests in config_file. See Also -------- run_unit_tests run_examples run_system run_release ''' # Print test set welcome print('\n-'.ljust(OUTPUT_WIDTH + 1, '-')) print('Running Science Tests') print('-'.ljust(OUTPUT_WIDTH, '-')) # Get setup config = read_configobj(config_file) # drop invalid driver tests config = drop_tests(config, driver) test_results = OrderedDict() # Run individual tests for i, (test_type, test_dict) in enumerate(config.items()): # print out status info print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()), test_type)) # Setup directories for test dirs = setup_test_dirs(test_type, out_dir, mkdirs=['results', 'state', 'logs', 'plots']) # read template global parameter file infile = os.path.join(test_dir, 'science', test_dict['global_parameter_file']) with open(infile, 'r') as global_file: global_param = global_file.read() # create template string s = string.Template(global_param) # fill in global parameter options global_param = s.safe_substitute(test_data_dir=science_test_data_dir, test_dir=test_dir, result_dir=dirs['results'], state_dir=dirs['state'], testname=test_type, test_root=test_dir) test_global_file = os.path.join( dirs['test'], '{0}_globalparam.txt'.format(test_type)) # write global parameter file with open(test_global_file, 'w') as f: f.write(global_param) # Get optional kwargs for run executable run_kwargs = pop_run_kwargs(test_dict) # run VIC test_complete = False test_passed = False test_comment = '' error_message = '' try: # Run the VIC simulation returncode = vic_exe.run(test_global_file, logdir=dirs['logs'], **run_kwargs) test_complete = True # Check return code check_returncode(vic_exe) # check output files (different tests depending on driver) if test_dict['check']: fnames = glob.glob(os.path.join(dirs['results'], '*')) # Check that the simulation completed for all grid cells if 'complete' in test_dict['check'] and driver == 'classic': test_classic_driver_all_complete(fnames) # check for nans in all example files if 'output_file_nans' in test_dict['check']: if driver == 'classic': test_classic_driver_no_output_file_nans(fnames) elif driver == 'image': domain_file = os.path.join(test_data_dir, test_dict['domain_file']) test_image_driver_no_output_file_nans( fnames, domain_file) else: raise ValueError('unknown driver') # plot science test results plot_science_tests(test_dict['driver'], test_type, science_test_data_dir, dirs['results'], dirs['plots'], test_dict['plots'], test_dict['compare_data'], nproc=nproc) # if we got this far, the test passed. test_passed = True # Handle errors except Exception as e: test_comment, error_message = process_error(e, vic_exe) # record the test results test_results[test_type] = TestResults(test_type, test_complete=test_complete, passed=test_passed, comment=test_comment, error_message=error_message, returncode=returncode) print('-'.ljust(OUTPUT_WIDTH, '-')) print('Finished testing science tests.') print('-'.ljust(OUTPUT_WIDTH, '-')) return test_results
#!/usr/local/anaconda/bin/python ''' This scripts prepares a xmask file for Lohmann routine, using consistant method as in the inverse routing method (i.e., using haversine formula) ''' import numpy as np import argparse from tonic.io import read_configobj from da_utils import generate_xmask_for_route parser = argparse.ArgumentParser() parser.add_argument("--cfg", type=str, help="config file for this script") args = parser.parse_args() cfg = read_configobj(args.cfg) #========================================================# # Calculate xmask (i.e., flow distance) #========================================================# flow_distance = generate_xmask_for_route(cfg['INPUT']['fdir_path']) #========================================================# # Write to file #========================================================# f = open(cfg['OUTPUT']['output_xmask_path'], 'w') #--- Write header lines (copy from flow direction file) ---# f_fdir = open(cfg['INPUT']['fdir_path'], 'r') for i in range(6): line = f_fdir.readline().rstrip("\n") # read from flow direction file f.write(line + "\n") f_fdir.close() #--- Write xmask values ---# for i in range(len(flow_distance)):
import subprocess import string import pandas as pd import warnings warnings.filterwarnings('ignore') import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt from optimize_utils import read_RVIC_output, read_USGS_data, kge from tonic.io import read_configobj # ======================================================== # # Parse in parameters from MOCOM # ======================================================== # cfg = read_configobj(sys.argv[1]) # optimize cfg stor_dir = sys.argv[2] # .../run_ident/ soln_num = sys.argv[3] # the 5-digit parameter solution number out_plot_dir = sys.argv[4] # ======================================================== # # Parameter setting # ======================================================== # # --- RVIC --- # rvic_convolve_template = cfg['RVIC']['rvic_convolve_template'] # Time lag between VIC/RVIC forcing and USGS local time [hour] time_lag = cfg['RVIC']['time_lag'] # Site name in RVIC site = cfg['RVIC']['site'] # --- USGS streamflow data --- #
#!/bin/env python import matplotlib.pyplot as plt import os import numpy as np import pandas as pd import xarray as xr from tonic.io import read_config, read_configobj # read configs for filenames and diagnostics to run config_files = read_config(os.path.join(os.getcwd(), 'filenames.cfg')) config_diag = read_configobj(os.path.join(os.getcwd(), 'diagnostics.cfg')) # full list of l2x coupler fields l2x_vars = [ 'l2x_Sl_t', 'l2x_Sl_tref', 'l2x_Sl_qref', 'l2x_Sl_avsdr', 'l2x_Sl_anidr', 'l2x_Sl_avsdf', 'l2x_Sl_anidf', 'l2x_Sl_snowh', 'l2x_Sl_u10', 'l2x_Sl_fv', 'l2x_Sl_ram1', 'l2x_Sl_logz0', 'l2x_Fall_taux', 'l2x_Fall_tauy', 'l2x_Fall_lat', 'l2x_Fall_sen', 'l2x_Fall_lwup', 'l2x_Fall_evap', 'l2x_Fall_swnet', 'l2x_Fall_flxdst1', 'l2x_Fall_flxdst2', 'l2x_Fall_flxdst3', 'l2x_Fall_flxdst4', 'l2x_Flrl_rofliq', 'l2x_Flrl_rofice' ] l2x_units = [ 'K', 'K', 'g/g', 'fraction', 'fraction', 'fraction', 'fraction', 'm', 'm/s', 'm/s', 's/m', 'm', 'N/m2', 'N/m2', 'W/m2', 'W/m2', 'W/m2', 'kg/m2s', 'W/m2', 'm/s', 'm/s', 'm/s', 'm/s', 'kg/m2s', 'kg/m2s' ]
def run_science(config_file, vic_exe, science_test_data_dir, test_data_dir, out_dir, driver, nproc): '''Run science tests from config file Parameters ---------- config_file : str Configuration file for science tests. vic_exe : VIC (object) VIC executable object (see tonic documentation). science_test_data_dir: str Path to science test data sets (archived VIC runs and observations) test_data_dir : str Path to test data sets. out_dir : str Path to output location driver : {'classic', 'image'} Driver to run tests on. nproc : int Number of processors to use for science tests Returns ------- test_results : dict Test results for all tests in config_file. See Also -------- run_unit_tests run_examples run_system run_release ''' # Print test set welcome print('\n-'.ljust(OUTPUT_WIDTH + 1, '-')) print('Running Science Tests') print('-'.ljust(OUTPUT_WIDTH, '-')) # Get setup config = read_configobj(config_file) # drop invalid driver tests config = drop_tests(config, driver) test_results = OrderedDict() # Run individual tests for i, (test_type, test_dict) in enumerate(config.items()): # print out status info print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()), test_type)) # Setup directories for test dirs = setup_test_dirs(test_type, out_dir, mkdirs=['results', 'state', 'logs', 'plots']) # read template global parameter file infile = os.path.join(test_dir, 'science', test_dict['global_parameter_file']) with open(infile, 'r') as global_file: global_param = global_file.read() # create template string s = string.Template(global_param) # fill in global parameter options global_param = s.safe_substitute(test_data_dir=science_test_data_dir, test_dir=test_dir, result_dir=dirs['results'], state_dir=dirs['state'], testname=test_type, test_root=test_dir) test_global_file = os.path.join( dirs['test'], '{0}_globalparam.txt'.format(test_type)) # write global parameter file with open(test_global_file, 'w') as f: f.write(global_param) # Get optional kwargs for run executable run_kwargs = pop_run_kwargs(test_dict) # run VIC test_complete = False test_passed = False test_comment = '' error_message = '' try: # Run the VIC simulation returncode = vic_exe.run(test_global_file, logdir=dirs['logs'], **run_kwargs) test_complete = True # Check return code check_returncode(vic_exe) # check output files (different tests depending on driver) if test_dict['check']: fnames = glob.glob(os.path.join(dirs['results'], '*')) # Check that the simulation completed for all grid cells if 'complete' in test_dict['check'] and driver == 'classic': test_classic_driver_all_complete(fnames) # check for nans in all example files if 'output_file_nans' in test_dict['check']: if driver == 'classic': test_classic_driver_no_output_file_nans(fnames) elif driver == 'image': domain_file = os.path.join(test_data_dir, test_dict['domain_file']) test_image_driver_no_output_file_nans(fnames, domain_file) else: raise ValueError('unknown driver') # plot science test results plot_science_tests(test_dict['driver'], test_type, science_test_data_dir, dirs['results'], dirs['plots'], test_dict['plots'], test_dict['compare_data'], nproc=nproc) # if we got this far, the test passed. test_passed = True # Handle errors except Exception as e: test_comment, error_message = process_error(e, vic_exe) # record the test results test_results[test_type] = TestResults(test_type, test_complete=test_complete, passed=test_passed, comment=test_comment, error_message=error_message, returncode=returncode) print('-'.ljust(OUTPUT_WIDTH, '-')) print('Finished testing science tests.') print('-'.ljust(OUTPUT_WIDTH, '-')) return test_results
def run_system(config_file, dict_drivers, test_data_dir, out_dir): '''Run system tests from config file Parameters ---------- config_file : str Configuration file for system tests. dict_drivers : dict Keys: driver names {'classic', 'image'} Content: corresponding VIC executable object (see tonic documentation) test_data_dir : str Path to test data sets. out_dir : str Path to output location Returns ------- test_results : dict Test results for all tests in config_file. See Also -------- run_unit_tests run_examples run_science run_release ''' # Print test set welcome print('\n-'.ljust(OUTPUT_WIDTH + 1, '-')) print('Running System Tests') print('-'.ljust(OUTPUT_WIDTH, '-')) # Get setup config = read_configobj(config_file) # Process driver info if len(dict_drivers) == 1: # if single driver driver = list(dict_drivers.keys())[0] vic_exe = dict_drivers[driver] # Drop invalid driver tests if len(dict_drivers) == 1: # if single driver config = drop_tests(config, driver) else: # if multiple drivers config = drop_tests(config, list(dict_drivers)) test_results = OrderedDict() # Run individual system tests for i, (testname, test_dict) in enumerate(config.items()): # print out status info print('Running test {0}/{1}: {2}'.format(i + 1, len(config.items()), testname)) # Setup directories for test dirs = setup_test_dirs(testname, out_dir, mkdirs=['results', 'state', 'logs', 'plots']) # read template global parameter file dict_global_param = {} # --- if single driver --- # if len(dict_drivers) == 1: infile = os.path.join(test_dir, 'system', test_dict['global_parameter_file']) with open(infile, 'r') as global_file: dict_global_param[driver] = global_file.read() # --- if multiple drivers --- # else: for j, dr in enumerate(test_dict['driver']): infile = os.path.join(test_dir, 'system', test_dict['global_parameter_file'][j]) with open(infile, 'r') as global_file: dict_global_param[dr] = global_file.read() # If restart test, prepare running periods if 'exact_restart' in test_dict['check']: if len(dict_drivers) > 1: raise ValueError('Only support single driver for restart' 'tests!') global_param = dict_global_param[driver] # () Find STATE_FORMAT option for later use if driver == 'classic': state_format = find_global_param_value(global_param, 'STATE_FORMAT') # (2) Prepare running periods and initial state file info for # restart test run_periods = prepare_restart_run_periods( test_dict['restart'], dirs['state']) # If mpi test, prepare a list of number of processors to be run elif 'mpi' in test_dict['check']: if len(dict_drivers) > 1: raise ValueError('Only support single driver for MPI' 'tests!') if not isinstance(test_dict['mpi']['n_proc'], list): raise ValueError('Need at least two values in n_proc to run' 'mpi test!') list_n_proc = test_dict['mpi']['n_proc'] # create template string dict_s = {} for dr, global_param in dict_global_param.items(): dict_s[dr] = string.Template(global_param) # fill in global parameter options # --- if restart test, multiple runs --- # if 'exact_restart' in test_dict['check']: s = dict_s[driver] # Set up subdirectories and fill in global parameter options # for restart testing list_global_param =\ setup_subdirs_and_fill_in_global_param_restart_test( s, run_periods, driver, dirs['results'], dirs['state'], test_data_dir) # --- if mpi test, multiple runs --- # elif 'mpi' in test_dict['check']: s = dict_s[driver] # Set up subdirectories and output directories in global file for # multiprocessor testing list_global_param = \ setup_subdirs_and_fill_in_global_param_mpi_test( s, list_n_proc, dirs['results'], dirs['state'], test_data_dir) # --- if driver-match test, one run for each driver --- # elif 'driver_match' in test_dict['check']: # Set up subdirectories and output directories in global file for # driver-match testing dict_global_param = \ setup_subdirs_and_fill_in_global_param_driver_match_test( dict_s, dirs['results'], dirs['state'], test_data_dir) # --- else, single run --- # else: if len(dict_drivers) > 1: raise RuntimeError('Only support single driver for test' '{}!'.format(testname)) s = dict_s[driver] global_param = s.safe_substitute(test_data_dir=test_data_dir, result_dir=dirs['results'], state_dir=dirs['state']) # replace global options from config file # --- extract global options to be substitute --- # if 'options' in test_dict: replacements = test_dict['options'] else: replacements = OrderedDict() # --- replace global options --- # # For the purpose of exact restart, if STATE_FORMAT is specified, # then record the specified value (instead of the one in the global # template file) if 'exact_restart' in test_dict['check']: if 'STATE_FORMAT' in replacements: state_format = replacements['STATE_FORMAT'] if 'exact_restart' in test_dict['check'] or\ 'mpi' in test_dict['check']: # if multiple runs for j, gp in enumerate(list_global_param): # save a copy of replacements for the next global file replacements_cp = replacements.copy() # replace global options for this global file list_global_param[j] = replace_global_values(gp, replacements) replacements = replacements_cp elif 'driver_match' in test_dict['check']: # if cross-driver runs for dr, gp in dict_global_param.items(): # save a copy of replacements for the next global file replacements_cp = replacements.copy() # replace global options for this global file dict_global_param[dr] = replace_global_values(gp, replacements) replacements = replacements_cp else: # if single run global_param = replace_global_values(global_param, replacements) # write global parameter file if 'exact_restart' in test_dict['check']: list_test_global_file = [] for j, gp in enumerate(list_global_param): test_global_file = os.path.join( dirs['test'], '{}_globalparam_{}_{}.txt'.format( testname, run_periods[j]['start_date'].strftime("%Y%m%d"), run_periods[j]['end_date'].strftime("%Y%m%d"))) list_test_global_file.append(test_global_file) with open(test_global_file, mode='w') as f: for line in gp: f.write(line) elif 'mpi' in test_dict['check']: list_test_global_file = [] for j, gp in enumerate(list_global_param): test_global_file = os.path.join( dirs['test'], '{}_globalparam_processors_{}.txt'.format( testname, list_n_proc[j])) list_test_global_file.append(test_global_file) with open(test_global_file, mode='w') as f: for line in gp: f.write(line) elif 'driver_match' in test_dict['check']: dict_test_global_file = {} for dr, gp in dict_global_param.items(): test_global_file = os.path.join( dirs['test'], '{}_globalparam_{}.txt'.format( testname, dr)) dict_test_global_file[dr] = test_global_file with open(test_global_file, mode='w') as f: for line in gp: f.write(line) else: test_global_file = os.path.join( dirs['test'], '{0}_globalparam.txt'.format(testname)) with open(test_global_file, mode='w') as f: for line in global_param: f.write(line) # Get optional kwargs for run executable run_kwargs = pop_run_kwargs(test_dict) # run VIC test_complete = False test_passed = False test_comment = '' error_message = '' try: if 'exact_restart' in test_dict['check']: for j, test_global_file in enumerate(list_test_global_file): returncode = vic_exe.run(test_global_file, logdir=dirs['logs'], **run_kwargs) # Check return code check_returncode(vic_exe, test_dict.pop('expected_retval', 0)) elif 'mpi' in test_dict['check']: for j, test_global_file in enumerate(list_test_global_file): # Overwrite mpi_proc in option kwargs n_proc = list_n_proc[j] if n_proc == 1: run_kwargs['mpi_proc'] = None else: run_kwargs['mpi_proc'] = list_n_proc[j] # Run VIC returncode = vic_exe.run(test_global_file, logdir=dirs['logs'], **run_kwargs) # Check return code check_returncode(vic_exe, test_dict.pop('expected_retval', 0)) elif 'driver_match' in test_dict['check']: for dr in dict_test_global_file.keys(): # Reset mpi_proc in option kwargs to None for classic # driver run if dr == 'classic': run_kwargs_classic = run_kwargs run_kwargs_classic['mpi_proc'] = None returncode = dict_drivers[dr].run( dict_test_global_file[dr], logdir=dirs['logs'], **run_kwargs_classic) else: returncode = dict_drivers[dr].run( dict_test_global_file[dr], logdir=dirs['logs'], **run_kwargs) # Check return code check_returncode(dict_drivers[dr], test_dict.pop('expected_retval', 0)) else: returncode = vic_exe.run(test_global_file, logdir=dirs['logs'], **run_kwargs) # Check return code check_returncode(vic_exe, test_dict.pop('expected_retval', 0)) test_complete = True # check output files (different tests depending on driver) if 'check' in test_dict: # Check that the simulation completed for all grid cells if 'complete' in test_dict['check']: if len(dict_drivers) > 1: raise RuntimeError('Only support single driver for ' 'complete check') fnames = glob.glob(os.path.join(dirs['results'], '*')) if driver == 'classic': test_classic_driver_all_complete(fnames) else: raise RuntimeError('complete check only supports ' 'classic driver') # check for nans in all example files if 'output_file_nans' in test_dict['check']: if len(dict_drivers) > 1: raise RuntimeError('Only support single driver for ' 'output_file_nans check') fnames = glob.glob(os.path.join(dirs['results'], '*')) if driver == 'classic': test_classic_driver_no_output_file_nans(fnames) elif driver == 'image': domain_file = os.path.join( test_data_dir, test_dict['domain_file']) test_image_driver_no_output_file_nans( fnames, domain_file) else: raise ValueError('unknown driver') # check for exact restarts if 'exact_restart' in test_dict['check']: check_exact_restart_fluxes(dirs['results'], driver, run_periods) if driver == 'classic': check_exact_restart_states(dirs['state'], driver, run_periods, state_format) elif driver == 'image': check_exact_restart_states(dirs['state'], driver, run_periods) else: raise ValueError('unknown driver') # check for multistream output if 'multistream' in test_dict['check']: if len(dict_drivers) > 1: raise ValueError('Only support single driver for ' 'multistream check') fnames = glob.glob(os.path.join(dirs['results'], '*')) if driver == 'classic': check_multistream_classic(fnames) elif driver == 'image': warnings.warn('Skipping multistream image driver test') # TODO: check_multistream_image(fnames) # check for mpi multiprocessor results if 'mpi' in test_dict['check']: check_mpi_fluxes(dirs['results'], list_n_proc) check_mpi_states(dirs['state'], list_n_proc) # check that results from different drivers match if 'driver_match' in test_dict['check']: check_drivers_match_fluxes(list(dict_drivers.keys()), dirs['results']) # if we got this far, the test passed. test_passed = True # Handle errors except Exception as e: for dr, exe in dict_drivers.items(): test_comment, error_message = process_error(e, exe) # record the test results test_results[testname] = TestResults(testname, test_complete=test_complete, passed=test_passed, comment=test_comment, error_message=error_message, returncode=returncode) print('-'.ljust(OUTPUT_WIDTH, '-')) print('Finished testing system tests.') print('-'.ljust(OUTPUT_WIDTH, '-')) return test_results