def _create_watch_parameters_and_locations(cover_dst, land_cover_frac_dst, soild_prop_dst, watch_driving_dataset, conf):
file_template = 'data/WATCH_2D/driving/{}.ncml'
for path, var, name, min, max in WATCH_DRIVING_DATA:
location = DrivingDatasetLocation()
location.base_url = file_template.format(path)
location.dataset_type = cover_dst
location.driving_dataset = watch_driving_dataset
location.var_name = var
watch_jules_parameters = [
[constants.JULES_PARAM_DRIVE_DATA_START, "'1901-01-01 00:00:00'"],
[constants.JULES_PARAM_DRIVE_DATA_PERIOD, "10800"],
[constants.JULES_PARAM_DRIVE_FILE, "'data/WATCH_2D/driving/%vv/%vv_%y4%m2.nc'"],
[constants.JULES_PARAM_DRIVE_NVARS, "8"],
[constants.JULES_PARAM_DRIVE_VAR,
"'pstar' 't' 'q' 'wind' 'lw_down' 'sw_down' "
"'tot_rain' 'tot_snow'"],
[constants.JULES_PARAM_DRIVE_VAR_NAME, "'PSurf' 'Tair' 'Qair' 'Wind' 'LWdown' "
"'SWdown' 'Rainf' 'Snowf'"],
[constants.JULES_PARAM_DRIVE_TPL_NAME, "'PSurf_WFD' 'Tair_WFD' "
"'Qair_WFD' 'Wind_WFD' 'LWdown_WFD' 'SWdown_WFD' "
"'Rainf_WFD_GPCC' 'Snowf_WFD_GPCC'"],
[constants.JULES_PARAM_DRIVE_INTERP,
"'i' 'i' 'i' 'i' 'nf' 'nf' 'nf' 'nf'"],
[constants.JULES_PARAM_DRIVE_Z1_TQ_IN, "2.0"],
[constants.JULES_PARAM_DRIVE_Z1_UV_IN, "10.0"],
[constants.JULES_PARAM_INPUT_GRID_IS_1D, ".false."],
[constants.JULES_PARAM_INPUT_GRID_NX, "720"],
[constants.JULES_PARAM_INPUT_GRID_NY, "280"],
[constants.JULES_PARAM_INPUT_GRID_X_DIM_NAME, "'Longitude'"],
[constants.JULES_PARAM_INPUT_GRID_Y_DIM_NAME, "'Latitude'"],
[constants.JULES_PARAM_INPUT_TIME_DIM_NAME, "'Time'"],
[constants.JULES_PARAM_INPUT_TYPE_DIM_NAME, "'pseudo'"],
[constants.JULES_PARAM_LATLON_FILE, ("'%s'" % WATCH_LATLON_FILE)],
[constants.JULES_PARAM_LATLON_LAT_NAME, "'Grid_lat'"],
[constants.JULES_PARAM_LATLON_LON_NAME, "'Grid_lon'"],
[constants.JULES_PARAM_LAND_FRAC_FILE, ("'%s'" % WATCH_LAND_FRAC_FILE)],
[constants.JULES_PARAM_LAND_FRAC_LAND_FRAC_NAME, "'land'"],
[constants.JULES_PARAM_SURF_HGT_ZERO_HEIGHT, ".true."],
[constants.JULES_PARAM_FRAC_FILE, ("'%s'" % WATCH_FRAC_FILE)],
[constants.JULES_PARAM_FRAC_NAME, "'frac'"],
[constants.JULES_PARAM_SOIL_PROPS_CONST_Z, ".true."],
[constants.JULES_PARAM_SOIL_PROPS_FILE, ("'%s'" % WATCH_SOIL_PROPS_FILE)],
[constants.JULES_PARAM_SOIL_PROPS_NVARS, "9"],
[constants.JULES_PARAM_SOIL_PROPS_VAR,
"'b' 'sathh' 'satcon' 'sm_sat' 'sm_crit' 'sm_wilt' 'hcap' 'hcon' 'albsoil'"],
[constants.JULES_PARAM_SOIL_PROPS_VAR_NAME,
"'bexp' 'sathh' 'satcon' 'vsat' 'vcrit' 'vwilt' 'hcap' 'hcon' 'albsoil'"],
[constants.JULES_PARAM_INITIAL_NVARS, "10"],
[constants.JULES_PARAM_INITIAL_VAR,
"'sthuf' 'canopy' 'snow_tile' 'rgrain' 'tstar_tile' 't_soil' 'cs' 'gs' 'lai' 'canht'"],
[constants.JULES_PARAM_INITIAL_USE_FILE,
".false. .false. .false. .false. .false. .false. .false. .false. .false. .false."],
[constants.JULES_PARAM_INITIAL_CONST_VAL,
"0.9 0.0 0.0 50.0 275.0 278.0 10.0 0.0 1.0 2.0"],
[constants.JULES_PARAM_POST_PROCESSING_ID, "1"]
]
if conf['full_data_range'].lower() == "true":
watch_jules_parameters.append([constants.JULES_PARAM_DRIVE_DATA_END, "'2001-12-31 21:00:00'"])
else:
watch_jules_parameters.append([constants.JULES_PARAM_DRIVE_DATA_END, "'1901-01-31 21:00:00'"])
model_run_service = ModelRunService()
for constant, value in watch_jules_parameters:
log.info(" adding parameter {}::{}".format(constant[0], constant[1]))
DrivingDatasetParameterValue(model_run_service, watch_driving_dataset, constant, value)
def setUp(self):
self.start_delta = datetime.timedelta(hours=1)
self.end_delta = datetime.timedelta(hours=2)
def _create_mock_dap_client(var_name):
def _gtia(date):
return date + self.start_delta
def _gtib(date):
return date - self.end_delta
def _data(lat, lon, date):
return len(var_name) * lat * lon + date
def _desc():
return var_name + "_desc"
def _lat_lon_index(lat, lon):
return lat, lon
def _time_index(date):
start = datetime.datetime(1900, 1, 1)
secs = (date - start).total_seconds()
return int(secs / (60 * 30))
mock_dap_client = MagicMock()
mock_dap_client.get_time_immediately_after = _gtia
mock_dap_client.get_time_immediately_before = _gtib
mock_dap_client.get_data_at = _data
mock_dap_client.get_longname = _desc
mock_dap_client.get_lat_lon_index = _lat_lon_index
mock_dap_client.get_time_index = _time_index
return mock_dap_client
def _create_dap_client(url):
if "file1" in url:
return _create_mock_dap_client("variable1")
elif "file2" in url:
return _create_mock_dap_client("v2")
mock_dap_client_factory = DapClientFactory()
mock_dap_client_factory.get_dap_client = MagicMock(side_effect=_create_dap_client)
self.download_helper = AsciiDownloadHelper("test_url", dap_client_factory=mock_dap_client_factory)
self.model_run_service = ModelRunService()
self.nvars = 2
self.period = 30 * 60
with session_scope(Session) as session:
driving_data = DrivingDataset()
driving_data.name = "Test Driving Dataset"
pv1 = DrivingDatasetParameterValue(self.model_run_service, driving_data,
constants.JULES_PARAM_DRIVE_DATA_PERIOD, self.period)
pv2 = DrivingDatasetParameterValue(self.model_run_service, driving_data,
constants.JULES_PARAM_DRIVE_NVARS, self.nvars)
pv3 = DrivingDatasetParameterValue(self.model_run_service, driving_data,
constants.JULES_PARAM_DRIVE_VAR, "'var1' 'var2'")
pv4 = DrivingDatasetParameterValue(self.model_run_service, driving_data,
constants.JULES_PARAM_DRIVE_INTERP, "'i' 'nf'")
ddl1 = DrivingDatasetLocation()
ddl1.base_url = "data/file1.nc"
ddl1.var_name = "var1"
ddl1.driving_dataset = driving_data
ddl2 = DrivingDatasetLocation()
ddl2.base_url = "data/file2.nc"
ddl2.var_name = "var2"
ddl2.driving_dataset = driving_data
session.add_all([driving_data])
self.driving_data = DatasetService().get_driving_dataset_by_id(driving_data.id)
def _create_chess_parameters_and_locations(cover_dst, land_cover_frac_dst, soild_prop_dst, driving_dataset, conf):
file_template = 'data/CHESS/driving/chess_{}_copy.ncml'
vars = ""
templates = ""
interps = ""
var_names = ""
for jules_var, var, template, interp, title, min, max in CHESS_DRIVING_DATA:
location = DrivingDatasetLocation()
location.base_url = file_template.format(template)
location.dataset_type = cover_dst
location.driving_dataset = driving_dataset
location.var_name = jules_var
vars += "'{}' ".format(jules_var)
templates += "'{}' ".format(template)
interps += "'{}' ".format(interp)
var_names += "'{}' ".format(var)
nvars = len(CHESS_DRIVING_DATA)
jules_parameters = [
[constants.JULES_PARAM_MODEL_LEVELS_ICE_INDEX, "-1"],
[constants.JULES_PARAM_MODEL_LEVELS_NNVG, "3"],
[constants.JULES_PARAM_FRAC_FILE, ("'%s'" % CHESS_FRAC_FILE)],
[constants.JULES_PARAM_FRAC_NAME, "'frac'"],
[constants.JULES_PARAM_SOIL_PROPS_NVARS, "9"],
[constants.JULES_PARAM_SOIL_PROPS_USE_FILE, ".true. .true. .true. .true. .true. .true. .true. .true. .false."],
[constants.JULES_PARAM_SOIL_PROPS_FILE, ("'%s'" % CHESS_SOIL_PROPS_FILE)],
[constants.JULES_PARAM_SOIL_PROPS_VAR,
"'b' 'sathh' 'satcon' 'sm_sat' 'sm_crit' 'sm_wilt' 'hcap' 'hcon' 'albsoil'"],
[constants.JULES_PARAM_SOIL_PROPS_VAR_NAME,
"'oneovernminusone' 'oneoveralpha' 'satcon' 'vsat' 'vcrit' 'vwilt' 'hcap' 'hcon' 'albsoil'"],
[constants.JULES_PARAM_SOIL_PROPS_CONST_VAL, "0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.15"],
[constants.JULES_PARAM_INITIAL_NVARS, "10"],
[constants.JULES_PARAM_INITIAL_VAR,
"'sthuf' 'canopy' 'snow_tile' 'rgrain' 'tstar_tile' 't_soil' 'cs' 'gs' 'lai' 'canht'"],
[constants.JULES_PARAM_INITIAL_USE_FILE,
".false. .false. .false. .false. .false. .false. .false. .false. .false. .false."],
[constants.JULES_PARAM_INITIAL_CONST_VAL,
"0.9 0.0 0.0 50.0 275.0 278.0 10.0 0.0 1.0 2.0"],
[constants.JULES_PARAM_DRIVE_DATA_START, "'1961-01-01 00:00:00'"],
[constants.JULES_PARAM_DRIVE_DATA_PERIOD, "86400"],
[constants.JULES_PARAM_DRIVE_FILE, "'data/CHESS/driving/chess_%vv_%y4%m2_copy.nc'"],
[constants.JULES_PARAM_DRIVE_NVARS, nvars],
[constants.JULES_PARAM_DRIVE_VAR, vars],
[constants.JULES_PARAM_DRIVE_VAR_NAME, var_names],
[constants.JULES_PARAM_DRIVE_TPL_NAME, templates],
[constants.JULES_PARAM_DRIVE_INTERP, interps],
[constants.JULES_PARAM_DRIVE_Z1_TQ_IN, "1.2"],
[constants.JULES_PARAM_DRIVE_Z1_UV_IN, "10.0"],
[constants.JULES_PARAM_DRIVE_L_DAILY_DISAGG, ".true."],
[constants.JULES_PARAM_DRIVE_L_DISAGG_RH, ".true."],
[constants.JULES_PARAM_DRIVE_PRECIP_DISAGG_METHOD, "3"],
[constants.JULES_PARAM_DRIVE_DIFF_FRAC_CONST, "0.4"],
[constants.JULES_PARAM_DRIVE_T_FOR_SNOW, "275.15"],
[constants.JULES_PARAM_DRIVE_T_FOR_CON_RAIN, "288.15"],
[constants.JULES_PARAM_DRIVE_DUR_CONV_RAIN, "7200.0"],
[constants.JULES_PARAM_DRIVE_DUR_LS_RAIN, "18000.0"],
[constants.JULES_PARAM_DRIVE_DUR_LS_SNOW, "18000.0"],
[constants.JULES_PARAM_INPUT_GRID_IS_1D, ".false."],
[constants.JULES_PARAM_INPUT_GRID_NX, "656"],
[constants.JULES_PARAM_INPUT_GRID_NY, "1057"],
[constants.JULES_PARAM_INPUT_GRID_X_DIM_NAME, "'x'"],
[constants.JULES_PARAM_INPUT_GRID_Y_DIM_NAME, "'y'"],
[constants.JULES_PARAM_INPUT_TIME_DIM_NAME, "'Time'"],
[constants.JULES_PARAM_INPUT_TYPE_DIM_NAME, "'z'"],
[constants.JULES_PARAM_INPUT_SOIL_DIM_NAME, "'z'"],
[constants.JULES_PARAM_INPUT_PFT_DIM_NAME, "'pft'"],
[constants.JULES_PARAM_LATLON_FILE, ("'%s'" % CHESS_LATLON_FILE)],
[constants.JULES_PARAM_LATLON_LAT_NAME, "'lat'"],
[constants.JULES_PARAM_LATLON_LON_NAME, "'lon'"],
[constants.JULES_PARAM_LAND_FRAC_FILE, ("'%s'" % CHESS_LAND_FRAC_FILE)],
[constants.JULES_PARAM_LAND_FRAC_LAND_FRAC_NAME, "'landfrac'"],
[constants.JULES_PARAM_SURF_HGT_ZERO_HEIGHT, ".true."],
[constants.JULES_PARAM_POST_PROCESSING_ID, "2"],
[constants.JULES_PARAM_SWITCHES_L_VG_SOIL, ".true."]
]
extra_pft_parameters = [
["pftname_io ", " 'BT' 'NT' 'grass' 'shrub' 'crop'"],
["canht_ft_io ", " 19.01 16.38 0.79 1.00 0.79"],
["lai_io ", " 5.0 4.0 2.0 1.0 2.0"],
["c3_io ", " 1 1 1 1 1"],
["orient_io ", " 0 0 0 0 0"],
["a_wl_io ", " 0.65 0.65 0.005 0.10 0.005"],
["a_ws_io ", " 10.00 10.00 1.00 10.00 1.00"],
["albsnc_max_io ", " 0.15 0.15 0.60 0.40 0.60"],
["albsnc_min_io ", " 0.30 0.30 0.80 0.80 0.80"],
["albsnf_max_io ", " 0.10 0.10 0.20 0.20 0.20"],
["dz0v_dh_io ", " 0.05 0.05 0.10 0.10 0.10"],
["catch0_io ", " 0.50 0.50 0.50 0.50 0.50"],
["dcatch_dlai_io ", " 0.05 0.05 0.05 0.05 0.05"],
["infil_f_io ", " 4.00 4.00 2.00 2.00 2.00"],
["kext_io ", " 0.50 0.50 0.50 0.50 0.50"],
["rootd_ft_io ", " 3.00 1.00 0.50 0.50 0.50"],
["z0hm_pft_io ", " 0.10 0.10 0.10 0.10 0.10"],
["alpha_io ", " 0.08 0.08 0.08 0.08 0.08"],
["alnir_io ", " 0.45 0.35 0.58 0.58 0.58"],
["alpar_io ", " 0.10 0.07 0.10 0.10 0.10"],
["b_wl_io ", " 1.667 1.667 1.667 1.667 1.667"],
["dgl_dm_io ", " 0.0 0.0 0.0 0.0 0.0"],
["dgl_dt_io ", " 9.0 9.0 0.0 9.0 0.0"],
["dqcrit_io ", " 0.090 0.060 0.100 0.100 0.100"],
["eta_sl_io ", " 0.01 0.01 0.01 0.01 0.01"],
["fd_io ", " 0.015 0.015 0.015 0.015 0.015"],
["fsmc_of_io ", " 0.00 0.00 0.00 0.00 0.00"],
["f0_io ", " 0.875 0.875 0.900 0.900 0.900"],
["g_leaf_0_io ", " 0.25 0.25 0.25 0.25 0.25"],
["glmin_io ", "1.0E-6 1.0E-6 1.0E-6 1.0E-6 1.0E-6"],
["kpar_io ", " 0.50 0.50 0.50 0.50 0.50"],
["neff_io ", "0.8e-3 0.8e-3 0.8e-3 0.8e-3 0.8e-3"],
["nl0_io ", " 0.040 0.030 0.060 0.030 0.060"],
["nr_nl_io ", " 1.00 1.00 1.00 1.00 1.00"],
["ns_nl_io ", " 0.10 0.10 1.00 0.10 1.00"],
["omega_io ", " 0.15 0.15 0.15 0.15 0.15"],
["omnir_io ", " 0.70 0.45 0.83 0.83 0.83"],
["r_grow_io ", " 0.25 0.25 0.25 0.25 0.25"],
["sigl_io ", "0.0375 0.1000 0.0250 0.0500 0.0250"],
["tleaf_of_io ", "273.15 243.15 258.15 243.15 258.15"],
["tlow_io ", " 0.0 -5.0 0.0 0.0 0.0"],
["tupp_io ", " 36.0 31.0 36.0 36.0 36.0"],
["emis_pft_io ", " 0.97 0.97 0.97 0.97 0.97"],
["fl_o3_ct_io ", " 1.6 1.6 5.0 1.6 5.0"],
["dfp_dcuo_io ", " 0.04 0.02 0.25 0.03 0.25"],
["ief_io ", " 35.0 12.0 16.0 20.0 16.0"],
["tef_io ", " 0.40 2.40 0.80 0.80 0.80"],
["mef_io ", " 0.60 0.90 0.60 0.57 0.60"],
["aef_io ", " 0.18 0.21 0.12 0.20 0.12"],
["z0hm_classic_pft_io ", " 0.1 0.1 0.1 0.1 0.1"],
["albsnf_maxu_io ", " 0.215 0.132 0.288 0.173 0.288"],
["albsnf_maxl_io ", " 0.095 0.059 0.128 0.077 0.128"],
["alniru_io ", " 0.68 0.53 0.87 0.87 0.87"],
["alnirl_io ", " 0.30 0.23 0.39 0.39 0.39"],
["alparu_io ", " 0.15 0.11 0.15 0.15 0.15"],
["alparl_io ", " 0.06 0.04 0.06 0.06 0.06"],
["omegau_io ", " 0.23 0.23 0.23 0.23 0.23"],
["omegal_io ", " 0.10 0.10 0.10 0.10 0.10"],
["omniru_io ", " 0.90 0.65 0.98 0.98 0.98"],
["omnirl_io ", " 0.50 0.30 0.53 0.53 0.53"]]
for name, value in extra_pft_parameters:
parameter_constant = [constants.JULES_NML_PFTPARM, name.strip()]
jules_parameters.append([parameter_constant, value])
if conf['full_data_range'].lower() == "true":
jules_parameters.append([constants.JULES_PARAM_DRIVE_DATA_END, "'2013-01-01 00:00:00'"])
else:
jules_parameters.append([constants.JULES_PARAM_DRIVE_DATA_END, "'1961-01-31 00:00:00'"])
model_run_service = ModelRunService()
for constant, value in jules_parameters:
log.info(" adding parameter {}::{}".format(constant[0], constant[1]))
DrivingDatasetParameterValue(model_run_service, driving_dataset, constant, value)