def test_isnobal_restart(self):
""" Test standard iSnobal with crash restart """
config = deepcopy(self.base_config)
config.raw_cfg['awsm master']['model_type'] = 'isnobal'
config.raw_cfg['awsm master']['make_nc'] = False
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
result = can_i_run_awsm(config)
# run again with restart
config = deepcopy(self.base_config)
config.raw_cfg['awsm master']['model_type'] = 'isnobal'
config.raw_cfg['awsm master']['make_nc'] = False
config.raw_cfg['isnobal restart']['restart_crash'] = True
config.raw_cfg['isnobal restart']['wyh_restart_output'] = 1464
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
self.assertTrue(result)
def test_pysnobal_ouput_all(make_clean, base_config_copy):
config = base_config_copy
config.raw_cfg['files'].update({'output_mode': 'all'})
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
# run PySnobal
status = PySnobal(config).run()
assert status
# load in the outputs
gold = pd.read_csv(
'pysnobal/tests/test_data_point/gold_csv/gold.pysnobal.all.csv',
index_col='date_time',
parse_dates=True)
gold.index = gold.index.tz_convert('MST')
new = pd.read_csv('pysnobal/tests/output/pysnobal_output.csv',
index_col='date_time',
parse_dates=True)
new.index = new.index.tz_convert('MST')
pd.testing.assert_frame_equal(gold, new)
def configure(cls):
config = cls.base_config_copy()
config.raw_cfg['gridded']['hrrr_load_method'] = 'timestep'
config.apply_recipes()
cls.run_config = cast_all_variables(config, config.mcfg)
def configure(cls):
config = cls.base_config_copy()
config.raw_cfg['system']['threading'] = False
config.apply_recipes()
cls.run_config = cast_all_variables(config, config.mcfg)
def test_landfire_200(self, mock_reproject):
config = self.base_config_copy()
config.raw_cfg['generate_topo'][
'vegetation_dataset'] = 'landfire_2.0.0' # noqa
config.raw_cfg['generate_topo'][
'vegetation_folder'] = '../../landfire/landfire_200' # noqa
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
gt = GenerateTopo(config_file=config)
gt.run()
self.assertTrue(mock_reproject.called)
self.assertTrue(mock_reproject.call_count == 1)
ds = xr.open_dataset(os.path.join(self.basin_dir, 'output', 'topo.nc'))
self.assertCountEqual(list(ds.coords.keys()), ['y', 'x'])
self.assertCountEqual(list(ds.keys()), [
'dem', 'mask', 'veg_height', 'veg_k', 'veg_tau', 'veg_type',
'projection'
])
self.compare_netcdf_files('landfire_200/topo.nc', 'topo.nc')
def configure(cls):
config = cls.base_config_copy()
config.raw_cfg['awsm master']['run_smrf'] = False
config.raw_cfg['awsm master']['model_type'] = 'smrf_ipysnobal'
config.raw_cfg['system']['threading'] = False
config.apply_recipes()
cls.run_config = cast_all_variables(config, config.mcfg)
def thread_config(cls):
config = cls.base_config_copy()
config.raw_cfg['system'].update(cls.THREAD_CONFIG)
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
return config
def change_variables(self, new_variables):
config = copy(self.base_config)
config.raw_cfg['output'].update({'variables': new_variables})
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
return config
def setUpClass(cls):
super().setUpClass()
config = cls.base_config_copy()
del config.raw_cfg['csv']
adj_config = {
'gridded': {
'data_type': 'hrrr_grib',
'hrrr_directory': './gridded/hrrr_test/',
},
'time': {
'start_date': '2018-07-22 16:00',
'end_date': '2018-07-22 20:00',
'time_zone': 'utc'
},
'system': {
'threading': False,
'log_file': './output/test.log'
},
'air_temp': {
'grid_local': True,
'grid_local_n': 25
},
'vapor_pressure': {
'grid_local': True,
'grid_local_n': 25
},
'precip': {
'grid_local': True,
'grid_local_n': 25,
'precip_temp_method': 'dew_point'
},
'wind': {
'wind_model': 'interp'
},
'thermal': {
'correct_cloud': True,
'correct_veg': True
},
'albedo': {
'grain_size': 300.0,
'max_grain': 2000.0
}
}
config.raw_cfg.update(adj_config)
# set the distribution to grid, thermal defaults will be fine
for v in cls.DIST_VARIABLES:
config.raw_cfg[v]['distribution'] = 'grid'
config.raw_cfg[v]['grid_mask'] = 'False'
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
cls.config = config
cls.gold_dir = cls.basin_dir.joinpath('gold_hrrr')
def configure(cls):
config = cls.base_config_copy()
config.raw_cfg['files']['init_file'] = './topo/init.nc'
config.raw_cfg['files']['init_type'] = 'netcdf'
config.apply_recipes()
cls.run_config = cast_all_variables(config, config.mcfg)
def test_load_timestep(self):
config = deepcopy(self.config)
config.raw_cfg['gridded']['hrrr_load_method'] = 'timestep'
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
run_smrf(config)
self.compare_hrrr_gold()
def test_load_timestep_threaded(self):
config = deepcopy(self.config)
config.raw_cfg['gridded']['hrrr_load_method'] = 'timestep'
config.raw_cfg['system']['threading'] = True
config.raw_cfg['system']['queue_max_values'] = 1
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
run_smrf(config)
self.compare_hrrr_gold()
def cast_recipes(self, config):
"""Cast the inicheck recipes
Arguments:
config {UserConfig} -- UserConfig object to modify
Returns:
UserConfig -- Modified UserConfig object
"""
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
return config
def configure(cls):
config = cls.base_config_copy()
config.raw_cfg['gridded']['hrrr_load_method'] = 'timestep'
config.raw_cfg['awsm master']['run_smrf'] = False
config.raw_cfg['awsm master']['model_type'] = 'smrf_ipysnobal'
config.raw_cfg['system']['threading'] = True
config.raw_cfg['files']['init_file'] = './topo/init.nc'
config.raw_cfg['files']['init_type'] = 'netcdf'
config.apply_recipes()
cls.run_config = cast_all_variables(config, config.mcfg)
def setUpClass(cls):
"""
Runs the short simulation over reynolds mountain east
"""
super().setUpClass()
config = cls.base_config_copy()
config.raw_cfg['system']['threading'] = False
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
cls._base_config = config
def test_pysnobal_netcdf(self):
""" Test PySnobal with netCDF Forcing """
config = deepcopy(self.base_config)
config.raw_cfg['awsm master']['make_in'] = False
config.raw_cfg['awsm master']['mask_isnobal'] = True
config.raw_cfg['ipysnobal']['forcing_data_type'] = 'netcdf'
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
self.assertIsNone(run_awsm(config))
def setUpClass(self):
super().setUpClass()
config = self.base_config_copy()
config.raw_cfg['generate_topo']['vegetation_dataset'] = 'landfire_2.0.0' # noqa
config.raw_cfg['generate_topo']['vegetation_folder'] = '../../landfire/landfire_200' # noqa
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
self.subject = Landfire200(config.cfg['generate_topo'])
os.makedirs(self.subject.temp_dir, exist_ok=True)
def test_smrf_pysnobal_thread(self):
""" Test smrf passing variables to PySnobal threaded """
config = deepcopy(self.base_config)
config.raw_cfg['awsm master']['run_smrf'] = False
config.raw_cfg['awsm master']['make_in'] = False
config.raw_cfg['awsm master']['model_type'] = 'smrf_ipysnobal'
config.raw_cfg['system']['threading'] = True
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
self.assertIsNone(run_awsm(config))
def test_pysnobal(self):
""" Test standard Pysnobal """
config = deepcopy(self.base_config)
# config.raw_cfg['files']['init_type'] = 'ipw_out'
# config.raw_cfg['files']['init_file'] = './RME/snow.2538'
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
# ensure that the recipes are used
self.assertTrue(config.cfg['awsm master']['model_type'] == 'ipysnobal')
self.assertIsNone(run_awsm(config))
def test_all_stations(self):
"""
Test using all stations
"""
# test the end date
config = self.base_config
config.raw_cfg['csv']['stations'] = ['RMESP', 'RME_176']
# apply the new recipies
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
self.assertIsInstance(run_smrf(config), SMRF)
def test_mysql_metadata_error(self):
""" test no metadata found """
config = deepcopy(self.base_config)
options = deepcopy(self.options)
config.raw_cfg['mysql'] = options
config.raw_cfg['mysql']['stations'] = ['NOT_STID', 'NOPE']
del config.raw_cfg['csv']
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
with self.assertRaises(Exception):
result = run_smrf(config)
def test_grid_netcdf(self):
""" Generic NetCDF loading """
config = deepcopy(self.base_config)
del config.raw_cfg['csv']
generic_grid = {
'data_type': 'netcdf',
'netcdf_file': './RME/gridded/netcdf_test.nc',
'zone_number': '11',
'zone_letter': 'N',
'air_temp': 'air_temp',
'vapor_pressure': 'vapor_pressure',
'precip': 'precip',
'wind_speed': 'wind_speed',
'wind_direction': 'wind_direction',
'thermal': 'thermal',
'cloud_factor': 'cloud_factor'
}
config.raw_cfg['gridded'] = generic_grid
config.raw_cfg['system']['time_out'] = '25'
config.raw_cfg['system']['queue_max_values'] = '2'
# Doesn't work with true
config.raw_cfg['system']['threading'] = 'False'
# set the distribution to grid, thermal defaults will be fine
for v in self.dist_variables:
config.raw_cfg[v]['distribution'] = 'grid'
config.raw_cfg[v]['grid_mask'] = 'False'
config.raw_cfg['thermal']['correct_cloud'] = 'False'
config.raw_cfg['thermal']['correct_veg'] = 'True'
# fix the time to that of the WRF_test.nc
config.raw_cfg['time']['start_date'] = '2015-03-03 00:00'
config.raw_cfg['time']['end_date'] = '2015-03-03 04:00'
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
# ensure that the recipes are used
self.assertTrue(
'station_adjust_for_undercatch' not in config.cfg['precip'].keys())
self.assertTrue(config.cfg['thermal']['correct_cloud'] == False)
self.assertTrue(config.cfg['thermal']['correct_veg'] == True)
result = can_i_run_smrf(config)
self.assertTrue(result)
def test_grid_wrf(self):
""" WRF NetCDF loading """
config = deepcopy(self.base_config)
del config.raw_cfg['csv']
adj_config = {
'gridded': {
'data_type': 'wrf',
'wrf_file': './gridded/WRF_test.nc',
},
'system': {
'threading': 'False',
'log_file': './output/log.txt'
},
'precip': {
'station_adjust_for_undercatch': 'False'
},
'wind': {
'wind_model': 'interp'
},
'thermal': {
'correct_cloud': 'False',
'correct_veg': 'True'
},
'time': {
'start_date': '2015-03-03 00:00',
'end_date': '2015-03-03 04:00'
}
}
config.raw_cfg.update(adj_config)
# set the distribution to grid, thermal defaults will be fine
for v in self.dist_variables:
config.raw_cfg[v]['grid_mask'] = 'False'
# fix the time to that of the WRF_test.nc
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
# ensure that the recipes are used
self.assertTrue(
'station_adjust_for_undercatch' not in config.cfg['precip'].keys())
self.assertFalse(config.cfg['thermal']['correct_cloud'])
self.assertTrue(config.cfg['thermal']['correct_veg'])
run_smrf(config)
def test_all_stations(self):
"""
Test using all stations
"""
# test the end date
config = deepcopy(self.base_config)
config.raw_cfg['csv']['stations'] = ['RMESP', 'RME_176']
# apply the new recipies
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
# test the base run with the config file
result = self.can_i_run_smrf(config)
assert result
def test_station_dates(self):
"""
Test the start date not in the data
"""
config = self.base_config
# Use dates not in the dataset, expecting an error
config.raw_cfg['time']['start_date'] = '1900-01-01 00:00'
config.raw_cfg['time']['end_date'] = '1900-02-01 00:00'
# apply the new recipes
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
with self.assertRaises(Exception):
run_smrf(config)
def test_grid_hrrr_local(self):
""" HRRR grib2 loading with local elevation gradient """
config = deepcopy(self.base_config)
del config.raw_cfg['csv']
hrrr_grid = {'data_type': 'hrrr_grib',
'hrrr_directory': './RME/gridded/hrrr_test/',
'zone_number': 11,
'zone_letter': 'N'}
config.raw_cfg['gridded'] = hrrr_grid
config.raw_cfg['system']['threading'] = 'False'
config.raw_cfg['system']['log_file'] = './output/log.txt'
# set the distribution to grid, thermal defaults will be fine
for v in self.dist_variables:
config.raw_cfg[v]['distribution'] = 'grid'
config.raw_cfg[v]['grid_mask'] = 'False'
# local gradient
config.raw_cfg['air_temp']['grid_local'] = 'True'
config.raw_cfg['air_temp']['grid_local_n'] = '25' # only 47 grid cells
config.raw_cfg['vapor_pressure']['grid_local'] = 'True'
# Only 47 grid cells in domain
config.raw_cfg['vapor_pressure']['grid_local_n'] = '25'
config.raw_cfg['precip']['grid_local'] = 'True'
config.raw_cfg['precip']['grid_local_n'] = '25'
config.raw_cfg['thermal']['correct_cloud'] = 'True'
config.raw_cfg['thermal']['correct_veg'] = 'True'
# fix the time to that of the WRF_test.nc
config.raw_cfg['time']['start_date'] = '2018-07-22 16:00'
config.raw_cfg['time']['end_date'] = '2018-07-22 20:00'
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
# ensure that the recipes are used
self.assertTrue('station_adjust_for_undercatch' not in config.cfg['precip'].keys())
self.assertTrue(config.cfg['thermal']['correct_cloud'] == True)
self.assertTrue(config.cfg['thermal']['correct_veg'] == True)
result = can_i_run_smrf(config)
self.assertTrue(result)
def test_mysql_data_w_client(self):
"""
Run SMRF with MYSQL data from client, also can only be run from inside
NWRC.
"""
# test a succesful run specifiying client
config = deepcopy(self.base_config)
options = deepcopy(self.options)
config.raw_cfg['mysql'] = options
config.raw_cfg['mysql']['client'] = 'RME_test'
del config.raw_cfg['csv']
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
result = can_i_run_smrf(config)
assert result
def test_isnobal(self):
""" Test standard iSnobal """
config = deepcopy(self.base_config)
config.raw_cfg['awsm master']['make_nc'] = True
config.raw_cfg['awsm master']['mask_isnobal'] = True
config.raw_cfg['awsm master']['model_type'] = 'isnobal'
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
# ensure that the recipes are used
self.assertTrue(config.cfg['awsm master']['model_type'] == 'isnobal')
result = can_i_run_awsm(config)
self.assertTrue(result)
def test_mysql_data_error(self):
""" test no data found """
config = deepcopy(self.base_config)
options = deepcopy(self.options)
config.raw_cfg['mysql'] = options
config.raw_cfg['mysql']['stations'] = ['RMESP', 'RME_176']
del config.raw_cfg['csv']
# wrong time
config.raw_cfg['time']['start_date'] = '1900-01-01 00:00'
config.raw_cfg['time']['end_date'] = '1900-02-01 00:00'
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
with self.assertRaises(Exception):
result = run_smrf(config)
def update_config(self, update, config=None):
"""Update the config file with a dictionary of items
Arguments:
update {dict} -- dict of section updates
Keyword Arguments:
config {UserConfig} -- UserConfig object or copy
the base config (default: {None})
"""
if config is None:
config = deepcopy(self.base_config)
config.raw_cfg.update(update)
config.apply_recipes()
config = cast_all_variables(config, config.mcfg)
return config