def create_two_driving_datasets(self): """ Creates two driving datasets with datasets, parameter values etc set up :return: nothing """ with session_scope(Session) as session: self.create_driving_dataset(session) model_run_service = ModelRunService() driving2 = DrivingDataset() driving2.name = "driving2" driving2.description = "driving 2 description" driving2.geographic_region = 'Global' driving2.spatial_resolution = 'Half degree' driving2.temporal_resolution = '3 Hours' driving2.boundary_lat_north = 85 driving2.boundary_lat_south = -90 driving2.boundary_lon_west = -180 driving2.boundary_lon_east = 180 driving2.time_start = datetime.datetime(1901, 1, 1, 0, 0, 0) driving2.time_end = datetime.datetime(2001, 1, 1, 0, 0, 0) driving2.view_order_index = 200 driving2.usage_order_index = 2 driving2.is_restricted_to_admins = False location3 = DrivingDatasetLocation() location3.base_url = "base_url3" location3.driving_dataset = driving2 val = f90_helper.python_to_f90_str(8 * ["i"]) pv1 = DrivingDatasetParameterValue(model_run_service, driving2, constants.JULES_PARAM_DRIVE_INTERP, val) val = f90_helper.python_to_f90_str(3600) pv2 = DrivingDatasetParameterValue(model_run_service, driving2, constants.JULES_PARAM_DRIVE_DATA_PERIOD, val) val = f90_helper.python_to_f90_str("data/driving2/frac.nc") pv3 = DrivingDatasetParameterValue(model_run_service, driving2, constants.JULES_PARAM_FRAC_FILE, val) val = f90_helper.python_to_f90_str("frac2") pv4 = DrivingDatasetParameterValue(model_run_service, driving2, constants.JULES_PARAM_FRAC_NAME, val) val = f90_helper.python_to_f90_str(['b', 'sathh', 'satcon', 'sm_sat', 'sm_crit', 'sm_wilt', 'hcap', 'hcon', 'albsoil']) pv5 = DrivingDatasetParameterValue(model_run_service, driving2, constants.JULES_PARAM_SOIL_PROPS_VAR, val) val = f90_helper.python_to_f90_str(['bexp', 'sathh', 'satcon', 'vsat', 'vcrit', 'vwilt', 'hcap', 'hcon', 'albsoil']) pv6 = DrivingDatasetParameterValue(model_run_service, driving2, constants.JULES_PARAM_SOIL_PROPS_VAR_NAME, val) val = f90_helper.python_to_f90_str(9) pv7 = DrivingDatasetParameterValue(model_run_service, driving2, constants.JULES_PARAM_SOIL_PROPS_NVARS, val) val = f90_helper.python_to_f90_str("data/WATCH_2D/ancils/soil_igbp_bc_watch_0p5deg_capUM6.6_2D.nc") pv7 = DrivingDatasetParameterValue(model_run_service, driving2, constants.JULES_PARAM_SOIL_PROPS_FILE, val) session.add(driving2) session.commit()
def setUp(self): self.start_delta = datetime.timedelta(hours=1) self.end_delta = datetime.timedelta(hours=2) def _create_mock_dap_client(var_name): def _gtia(date): return date + self.start_delta def _gtib(date): return date - self.end_delta def _data(lat, lon, date): return len(var_name) * lat * lon + date def _desc(): return var_name + "_desc" def _lat_lon_index(lat, lon): return lat, lon def _time_index(date): start = datetime.datetime(1900, 1, 1) secs = (date - start).total_seconds() return int(secs / (60 * 30)) mock_dap_client = MagicMock() mock_dap_client.get_time_immediately_after = _gtia mock_dap_client.get_time_immediately_before = _gtib mock_dap_client.get_data_at = _data mock_dap_client.get_longname = _desc mock_dap_client.get_lat_lon_index = _lat_lon_index mock_dap_client.get_time_index = _time_index return mock_dap_client def _create_dap_client(url): if "file1" in url: return _create_mock_dap_client("variable1") elif "file2" in url: return _create_mock_dap_client("v2") mock_dap_client_factory = DapClientFactory() mock_dap_client_factory.get_dap_client = MagicMock(side_effect=_create_dap_client) self.download_helper = AsciiDownloadHelper("test_url", dap_client_factory=mock_dap_client_factory) self.model_run_service = ModelRunService() self.nvars = 2 self.period = 30 * 60 with session_scope(Session) as session: driving_data = DrivingDataset() driving_data.name = "Test Driving Dataset" pv1 = DrivingDatasetParameterValue(self.model_run_service, driving_data, constants.JULES_PARAM_DRIVE_DATA_PERIOD, self.period) pv2 = DrivingDatasetParameterValue(self.model_run_service, driving_data, constants.JULES_PARAM_DRIVE_NVARS, self.nvars) pv3 = DrivingDatasetParameterValue(self.model_run_service, driving_data, constants.JULES_PARAM_DRIVE_VAR, "'var1' 'var2'") pv4 = DrivingDatasetParameterValue(self.model_run_service, driving_data, constants.JULES_PARAM_DRIVE_INTERP, "'i' 'nf'") ddl1 = DrivingDatasetLocation() ddl1.base_url = "data/file1.nc" ddl1.var_name = "var1" ddl1.driving_dataset = driving_data ddl2 = DrivingDatasetLocation() ddl2.base_url = "data/file2.nc" ddl2.var_name = "var2" ddl2.driving_dataset = driving_data session.add_all([driving_data]) self.driving_data = DatasetService().get_driving_dataset_by_id(driving_data.id)
def _create_watch_parameters_and_locations(cover_dst, land_cover_frac_dst, soild_prop_dst, watch_driving_dataset, conf): file_template = 'data/WATCH_2D/driving/{}.ncml' for path, var, name, min, max in WATCH_DRIVING_DATA: location = DrivingDatasetLocation() location.base_url = file_template.format(path) location.dataset_type = cover_dst location.driving_dataset = watch_driving_dataset location.var_name = var watch_jules_parameters = [ [constants.JULES_PARAM_DRIVE_DATA_START, "'1901-01-01 00:00:00'"], [constants.JULES_PARAM_DRIVE_DATA_PERIOD, "10800"], [constants.JULES_PARAM_DRIVE_FILE, "'data/WATCH_2D/driving/%vv/%vv_%y4%m2.nc'"], [constants.JULES_PARAM_DRIVE_NVARS, "8"], [constants.JULES_PARAM_DRIVE_VAR, "'pstar' 't' 'q' 'wind' 'lw_down' 'sw_down' " "'tot_rain' 'tot_snow'"], [constants.JULES_PARAM_DRIVE_VAR_NAME, "'PSurf' 'Tair' 'Qair' 'Wind' 'LWdown' " "'SWdown' 'Rainf' 'Snowf'"], [constants.JULES_PARAM_DRIVE_TPL_NAME, "'PSurf_WFD' 'Tair_WFD' " "'Qair_WFD' 'Wind_WFD' 'LWdown_WFD' 'SWdown_WFD' " "'Rainf_WFD_GPCC' 'Snowf_WFD_GPCC'"], [constants.JULES_PARAM_DRIVE_INTERP, "'i' 'i' 'i' 'i' 'nf' 'nf' 'nf' 'nf'"], [constants.JULES_PARAM_DRIVE_Z1_TQ_IN, "2.0"], [constants.JULES_PARAM_DRIVE_Z1_UV_IN, "10.0"], [constants.JULES_PARAM_INPUT_GRID_IS_1D, ".false."], [constants.JULES_PARAM_INPUT_GRID_NX, "720"], [constants.JULES_PARAM_INPUT_GRID_NY, "280"], [constants.JULES_PARAM_INPUT_GRID_X_DIM_NAME, "'Longitude'"], [constants.JULES_PARAM_INPUT_GRID_Y_DIM_NAME, "'Latitude'"], [constants.JULES_PARAM_INPUT_TIME_DIM_NAME, "'Time'"], [constants.JULES_PARAM_INPUT_TYPE_DIM_NAME, "'pseudo'"], [constants.JULES_PARAM_LATLON_FILE, ("'%s'" % WATCH_LATLON_FILE)], [constants.JULES_PARAM_LATLON_LAT_NAME, "'Grid_lat'"], [constants.JULES_PARAM_LATLON_LON_NAME, "'Grid_lon'"], [constants.JULES_PARAM_LAND_FRAC_FILE, ("'%s'" % WATCH_LAND_FRAC_FILE)], [constants.JULES_PARAM_LAND_FRAC_LAND_FRAC_NAME, "'land'"], [constants.JULES_PARAM_SURF_HGT_ZERO_HEIGHT, ".true."], [constants.JULES_PARAM_FRAC_FILE, ("'%s'" % WATCH_FRAC_FILE)], [constants.JULES_PARAM_FRAC_NAME, "'frac'"], [constants.JULES_PARAM_SOIL_PROPS_CONST_Z, ".true."], [constants.JULES_PARAM_SOIL_PROPS_FILE, ("'%s'" % WATCH_SOIL_PROPS_FILE)], [constants.JULES_PARAM_SOIL_PROPS_NVARS, "9"], [constants.JULES_PARAM_SOIL_PROPS_VAR, "'b' 'sathh' 'satcon' 'sm_sat' 'sm_crit' 'sm_wilt' 'hcap' 'hcon' 'albsoil'"], [constants.JULES_PARAM_SOIL_PROPS_VAR_NAME, "'bexp' 'sathh' 'satcon' 'vsat' 'vcrit' 'vwilt' 'hcap' 'hcon' 'albsoil'"], [constants.JULES_PARAM_INITIAL_NVARS, "10"], [constants.JULES_PARAM_INITIAL_VAR, "'sthuf' 'canopy' 'snow_tile' 'rgrain' 'tstar_tile' 't_soil' 'cs' 'gs' 'lai' 'canht'"], [constants.JULES_PARAM_INITIAL_USE_FILE, ".false. .false. .false. .false. .false. .false. .false. .false. .false. .false."], [constants.JULES_PARAM_INITIAL_CONST_VAL, "0.9 0.0 0.0 50.0 275.0 278.0 10.0 0.0 1.0 2.0"], [constants.JULES_PARAM_POST_PROCESSING_ID, "1"] ] if conf['full_data_range'].lower() == "true": watch_jules_parameters.append([constants.JULES_PARAM_DRIVE_DATA_END, "'2001-12-31 21:00:00'"]) else: watch_jules_parameters.append([constants.JULES_PARAM_DRIVE_DATA_END, "'1901-01-31 21:00:00'"]) model_run_service = ModelRunService() for constant, value in watch_jules_parameters: log.info(" adding parameter {}::{}".format(constant[0], constant[1])) DrivingDatasetParameterValue(model_run_service, watch_driving_dataset, constant, value)
def _create_chess_parameters_and_locations(cover_dst, land_cover_frac_dst, soild_prop_dst, driving_dataset, conf): file_template = 'data/CHESS/driving/chess_{}_copy.ncml' vars = "" templates = "" interps = "" var_names = "" for jules_var, var, template, interp, title, min, max in CHESS_DRIVING_DATA: location = DrivingDatasetLocation() location.base_url = file_template.format(template) location.dataset_type = cover_dst location.driving_dataset = driving_dataset location.var_name = jules_var vars += "'{}' ".format(jules_var) templates += "'{}' ".format(template) interps += "'{}' ".format(interp) var_names += "'{}' ".format(var) nvars = len(CHESS_DRIVING_DATA) jules_parameters = [ [constants.JULES_PARAM_MODEL_LEVELS_ICE_INDEX, "-1"], [constants.JULES_PARAM_MODEL_LEVELS_NNVG, "3"], [constants.JULES_PARAM_FRAC_FILE, ("'%s'" % CHESS_FRAC_FILE)], [constants.JULES_PARAM_FRAC_NAME, "'frac'"], [constants.JULES_PARAM_SOIL_PROPS_NVARS, "9"], [constants.JULES_PARAM_SOIL_PROPS_USE_FILE, ".true. .true. .true. .true. .true. .true. .true. .true. .false."], [constants.JULES_PARAM_SOIL_PROPS_FILE, ("'%s'" % CHESS_SOIL_PROPS_FILE)], [constants.JULES_PARAM_SOIL_PROPS_VAR, "'b' 'sathh' 'satcon' 'sm_sat' 'sm_crit' 'sm_wilt' 'hcap' 'hcon' 'albsoil'"], [constants.JULES_PARAM_SOIL_PROPS_VAR_NAME, "'oneovernminusone' 'oneoveralpha' 'satcon' 'vsat' 'vcrit' 'vwilt' 'hcap' 'hcon' 'albsoil'"], [constants.JULES_PARAM_SOIL_PROPS_CONST_VAL, "0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.15"], [constants.JULES_PARAM_INITIAL_NVARS, "10"], [constants.JULES_PARAM_INITIAL_VAR, "'sthuf' 'canopy' 'snow_tile' 'rgrain' 'tstar_tile' 't_soil' 'cs' 'gs' 'lai' 'canht'"], [constants.JULES_PARAM_INITIAL_USE_FILE, ".false. .false. .false. .false. .false. .false. .false. .false. .false. .false."], [constants.JULES_PARAM_INITIAL_CONST_VAL, "0.9 0.0 0.0 50.0 275.0 278.0 10.0 0.0 1.0 2.0"], [constants.JULES_PARAM_DRIVE_DATA_START, "'1961-01-01 00:00:00'"], [constants.JULES_PARAM_DRIVE_DATA_PERIOD, "86400"], [constants.JULES_PARAM_DRIVE_FILE, "'data/CHESS/driving/chess_%vv_%y4%m2_copy.nc'"], [constants.JULES_PARAM_DRIVE_NVARS, nvars], [constants.JULES_PARAM_DRIVE_VAR, vars], [constants.JULES_PARAM_DRIVE_VAR_NAME, var_names], [constants.JULES_PARAM_DRIVE_TPL_NAME, templates], [constants.JULES_PARAM_DRIVE_INTERP, interps], [constants.JULES_PARAM_DRIVE_Z1_TQ_IN, "1.2"], [constants.JULES_PARAM_DRIVE_Z1_UV_IN, "10.0"], [constants.JULES_PARAM_DRIVE_L_DAILY_DISAGG, ".true."], [constants.JULES_PARAM_DRIVE_L_DISAGG_RH, ".true."], [constants.JULES_PARAM_DRIVE_PRECIP_DISAGG_METHOD, "3"], [constants.JULES_PARAM_DRIVE_DIFF_FRAC_CONST, "0.4"], [constants.JULES_PARAM_DRIVE_T_FOR_SNOW, "275.15"], [constants.JULES_PARAM_DRIVE_T_FOR_CON_RAIN, "288.15"], [constants.JULES_PARAM_DRIVE_DUR_CONV_RAIN, "7200.0"], [constants.JULES_PARAM_DRIVE_DUR_LS_RAIN, "18000.0"], [constants.JULES_PARAM_DRIVE_DUR_LS_SNOW, "18000.0"], [constants.JULES_PARAM_INPUT_GRID_IS_1D, ".false."], [constants.JULES_PARAM_INPUT_GRID_NX, "656"], [constants.JULES_PARAM_INPUT_GRID_NY, "1057"], [constants.JULES_PARAM_INPUT_GRID_X_DIM_NAME, "'x'"], [constants.JULES_PARAM_INPUT_GRID_Y_DIM_NAME, "'y'"], [constants.JULES_PARAM_INPUT_TIME_DIM_NAME, "'Time'"], [constants.JULES_PARAM_INPUT_TYPE_DIM_NAME, "'z'"], [constants.JULES_PARAM_INPUT_SOIL_DIM_NAME, "'z'"], [constants.JULES_PARAM_INPUT_PFT_DIM_NAME, "'pft'"], [constants.JULES_PARAM_LATLON_FILE, ("'%s'" % CHESS_LATLON_FILE)], [constants.JULES_PARAM_LATLON_LAT_NAME, "'lat'"], [constants.JULES_PARAM_LATLON_LON_NAME, "'lon'"], [constants.JULES_PARAM_LAND_FRAC_FILE, ("'%s'" % CHESS_LAND_FRAC_FILE)], [constants.JULES_PARAM_LAND_FRAC_LAND_FRAC_NAME, "'landfrac'"], [constants.JULES_PARAM_SURF_HGT_ZERO_HEIGHT, ".true."], [constants.JULES_PARAM_POST_PROCESSING_ID, "2"], [constants.JULES_PARAM_SWITCHES_L_VG_SOIL, ".true."] ] extra_pft_parameters = [ ["pftname_io ", " 'BT' 'NT' 'grass' 'shrub' 'crop'"], ["canht_ft_io ", " 19.01 16.38 0.79 1.00 0.79"], ["lai_io ", " 5.0 4.0 2.0 1.0 2.0"], ["c3_io ", " 1 1 1 1 1"], ["orient_io ", " 0 0 0 0 0"], ["a_wl_io ", " 0.65 0.65 0.005 0.10 0.005"], ["a_ws_io ", " 10.00 10.00 1.00 10.00 1.00"], ["albsnc_max_io ", " 0.15 0.15 0.60 0.40 0.60"], ["albsnc_min_io ", " 0.30 0.30 0.80 0.80 0.80"], ["albsnf_max_io ", " 0.10 0.10 0.20 0.20 0.20"], ["dz0v_dh_io ", " 0.05 0.05 0.10 0.10 0.10"], ["catch0_io ", " 0.50 0.50 0.50 0.50 0.50"], ["dcatch_dlai_io ", " 0.05 0.05 0.05 0.05 0.05"], ["infil_f_io ", " 4.00 4.00 2.00 2.00 2.00"], ["kext_io ", " 0.50 0.50 0.50 0.50 0.50"], ["rootd_ft_io ", " 3.00 1.00 0.50 0.50 0.50"], ["z0hm_pft_io ", " 0.10 0.10 0.10 0.10 0.10"], ["alpha_io ", " 0.08 0.08 0.08 0.08 0.08"], ["alnir_io ", " 0.45 0.35 0.58 0.58 0.58"], ["alpar_io ", " 0.10 0.07 0.10 0.10 0.10"], ["b_wl_io ", " 1.667 1.667 1.667 1.667 1.667"], ["dgl_dm_io ", " 0.0 0.0 0.0 0.0 0.0"], ["dgl_dt_io ", " 9.0 9.0 0.0 9.0 0.0"], ["dqcrit_io ", " 0.090 0.060 0.100 0.100 0.100"], ["eta_sl_io ", " 0.01 0.01 0.01 0.01 0.01"], ["fd_io ", " 0.015 0.015 0.015 0.015 0.015"], ["fsmc_of_io ", " 0.00 0.00 0.00 0.00 0.00"], ["f0_io ", " 0.875 0.875 0.900 0.900 0.900"], ["g_leaf_0_io ", " 0.25 0.25 0.25 0.25 0.25"], ["glmin_io ", "1.0E-6 1.0E-6 1.0E-6 1.0E-6 1.0E-6"], ["kpar_io ", " 0.50 0.50 0.50 0.50 0.50"], ["neff_io ", "0.8e-3 0.8e-3 0.8e-3 0.8e-3 0.8e-3"], ["nl0_io ", " 0.040 0.030 0.060 0.030 0.060"], ["nr_nl_io ", " 1.00 1.00 1.00 1.00 1.00"], ["ns_nl_io ", " 0.10 0.10 1.00 0.10 1.00"], ["omega_io ", " 0.15 0.15 0.15 0.15 0.15"], ["omnir_io ", " 0.70 0.45 0.83 0.83 0.83"], ["r_grow_io ", " 0.25 0.25 0.25 0.25 0.25"], ["sigl_io ", "0.0375 0.1000 0.0250 0.0500 0.0250"], ["tleaf_of_io ", "273.15 243.15 258.15 243.15 258.15"], ["tlow_io ", " 0.0 -5.0 0.0 0.0 0.0"], ["tupp_io ", " 36.0 31.0 36.0 36.0 36.0"], ["emis_pft_io ", " 0.97 0.97 0.97 0.97 0.97"], ["fl_o3_ct_io ", " 1.6 1.6 5.0 1.6 5.0"], ["dfp_dcuo_io ", " 0.04 0.02 0.25 0.03 0.25"], ["ief_io ", " 35.0 12.0 16.0 20.0 16.0"], ["tef_io ", " 0.40 2.40 0.80 0.80 0.80"], ["mef_io ", " 0.60 0.90 0.60 0.57 0.60"], ["aef_io ", " 0.18 0.21 0.12 0.20 0.12"], ["z0hm_classic_pft_io ", " 0.1 0.1 0.1 0.1 0.1"], ["albsnf_maxu_io ", " 0.215 0.132 0.288 0.173 0.288"], ["albsnf_maxl_io ", " 0.095 0.059 0.128 0.077 0.128"], ["alniru_io ", " 0.68 0.53 0.87 0.87 0.87"], ["alnirl_io ", " 0.30 0.23 0.39 0.39 0.39"], ["alparu_io ", " 0.15 0.11 0.15 0.15 0.15"], ["alparl_io ", " 0.06 0.04 0.06 0.06 0.06"], ["omegau_io ", " 0.23 0.23 0.23 0.23 0.23"], ["omegal_io ", " 0.10 0.10 0.10 0.10 0.10"], ["omniru_io ", " 0.90 0.65 0.98 0.98 0.98"], ["omnirl_io ", " 0.50 0.30 0.53 0.53 0.53"]] for name, value in extra_pft_parameters: parameter_constant = [constants.JULES_NML_PFTPARM, name.strip()] jules_parameters.append([parameter_constant, value]) if conf['full_data_range'].lower() == "true": jules_parameters.append([constants.JULES_PARAM_DRIVE_DATA_END, "'2013-01-01 00:00:00'"]) else: jules_parameters.append([constants.JULES_PARAM_DRIVE_DATA_END, "'1961-01-31 00:00:00'"]) model_run_service = ModelRunService() for constant, value in jules_parameters: log.info(" adding parameter {}::{}".format(constant[0], constant[1])) DrivingDatasetParameterValue(model_run_service, driving_dataset, constant, value)
def create_driving_dataset( self, session, jules_params=DrivingDatasetJulesParams(dataperiod=3600, var_interps=8 * ["i"]), is_restricted_to_admins=False, data_range_from = 10, data_range_to = 5): """ Create a driving dataset :param session: session to use :param jules_params: set of jules parameters :return: dataset """ model_run_service = ModelRunService() driving1 = DrivingDataset() driving1.name = "driving1" driving1.description = "driving 1 description" driving1.geographic_region = 'European' driving1.spatial_resolution = '1km' driving1.temporal_resolution = '24 hours' driving1.boundary_lat_north = 50 driving1.boundary_lat_south = -10 driving1.boundary_lon_west = -15 driving1.boundary_lon_east = 30 driving1.time_start = datetime.datetime(1979, 1, 1, 0, 0, 0) driving1.time_end = datetime.datetime(2010, 1, 1, 0, 0, 0) driving1.view_order_index = 100 driving1.is_restricted_to_admins = is_restricted_to_admins location1 = DrivingDatasetLocation() location1.base_url = "base_url" location1.dataset_type_id = 1 location1.driving_dataset = driving1 location2 = DrivingDatasetLocation() location2.base_url = "base_url2" location2.driving_dataset = driving1 location2.dataset_type_id = 1 jules_params.add_to_driving_dataset(model_run_service, driving1, session) val = f90_helper.python_to_f90_str(8 * ["i"]) pv1 = DrivingDatasetParameterValue(model_run_service, driving1, constants.JULES_PARAM_DRIVE_INTERP, val) val = f90_helper.python_to_f90_str(3600) pv2 = DrivingDatasetParameterValue(model_run_service, driving1, constants.JULES_PARAM_DRIVE_DATA_PERIOD, val) val = f90_helper.python_to_f90_str("data/driving1/frac.nc") pv3 = DrivingDatasetParameterValue(model_run_service, driving1, constants.JULES_PARAM_FRAC_FILE, val) val = f90_helper.python_to_f90_str("frac") pv4 = DrivingDatasetParameterValue(model_run_service, driving1, constants.JULES_PARAM_FRAC_NAME, val) session.add(driving1) session.commit() driving_data_filename_param_val = DrivingDatasetParameterValue( model_run_service, driving1, constants.JULES_PARAM_DRIVE_FILE, "'testFileName'") session.add(driving_data_filename_param_val) session.commit() return driving1