def test_multiple_timesteps(self): """ Test that the data has been reshaped correctly when there are multiple timesteps. The array contents are also checked. The output cube has only a single percentile, which is therefore demoted to a scalar coordinate. """ expected = np.array([ [[4.0, 4.71428571], [5.42857143, 6.14285714]], [[6.85714286, 7.57142857], [8.28571429, 9.0]], ]) cubelist = CubeList([]) for i, hour in enumerate([7, 8]): cubelist.append( set_up_percentile_cube( np.array([expected[i, :, :]], dtype=np.float32), np.array([50], dtype=np.float32), units="degC", time=datetime(2015, 11, 23, hour), frt=datetime(2015, 11, 23, 6), )) percentile_cube = cubelist.merge_cube() reshaped_array = restore_non_percentile_dimensions( percentile_cube.data.flatten(), next(percentile_cube.slices_over("percentile")), 1, ) self.assertArrayAlmostEqual(reshaped_array, expected)
def create_wind_percentile_cube(data=None, perc_values=None, name="wind_speed_of_gust"): """Create a cube with percentile coordinate and two time slices""" if perc_values is None: perc_values = [50.0] if data is None: data = np.zeros((len(perc_values), 2, 2, 2), dtype=np.float32) data[:, 0, :, :] = 1.0 data[:, 1, :, :] = 2.0 data_times = [datetime(2015, 11, 19, 0, 30), datetime(2015, 11, 19, 1, 30)] perc_cube = set_up_percentile_cube( data[:, 0, :, :], perc_values, name=name, units="m s-1", time=data_times[0], frt=datetime(2015, 11, 18, 21), ) cube = add_coordinate(perc_cube, data_times, "time", is_datetime=True, order=[1, 0, 2, 3]) cube.data = data return cube
def test_lots_of_input_percentiles(self): """ Test that the plugin returns an Iris.cube.Cube with the expected data values for the percentiles, if there are lots of thresholds. """ expected_data = np.array( [ [[11.0, 11.0, 11.0], [11.0, 11.0, 11.0], [11.0, 11.0, 11.0]], [[15.0, 15.0, 15.0], [15.0, 15.0, 15.0], [15.0, 15.0, 15.0]], [[19.0, 19.0, 19.0], [19.0, 19.0, 19.0], [19.0, 19.0, 19.0]], ] ) input_forecast_values = np.tile(np.linspace(10, 20, 30), (3, 3, 1)).T cube = set_up_percentile_cube( input_forecast_values.astype(np.float32), np.linspace(0, 100, 30).astype(np.float32), name="air_temperature", units="degC", ) result = Plugin()._interpolate_percentiles( cube, self.percentiles, self.bounds_pairing, self.perc_coord ) self.assertArrayAlmostEqual(result.data, expected_data)
def test_defaults(self): """Test default arguments produce cube with expected dimensions and metadata""" result = set_up_percentile_cube(self.data, self.percentiles) perc_coord = result.coord("percentile") self.assertArrayEqual(perc_coord.points, self.percentiles) self.assertEqual(perc_coord.units, "%") check_mandatory_standards(result)
def setUp(self): """Set up some "uncalibrated forecast" inputs""" attributes = { "title": "MOGREPS-UK Forecast", "source": "Met Office Unified Model", "institution": "Met Office", } forecast = np.array( [ np.full((3, 3), 10.4), np.full((3, 3), 10.8), np.full((3, 3), 10.1) ], dtype=np.float32, ) self.realizations = set_up_variable_cube(forecast, units="degC", attributes=attributes) percentiles = np.array( [ np.full((3, 3), 10.2), np.full((3, 3), 10.4), np.full((3, 3), 10.6) ], dtype=np.float32, ) self.percentiles = set_up_percentile_cube( percentiles, np.array([25, 50, 75], dtype=np.float32), units="degC", attributes=attributes, ) probabilities = np.array( [np.full((3, 3), 1), np.full((3, 3), 0.9), np.full((3, 3), 0)], dtype=np.float32, ) self.probabilities = set_up_probability_cube( probabilities, np.array([9, 10, 11], dtype=np.float32), threshold_units="degC", attributes=attributes, ) self.coefficients = build_coefficients_cubelist( self.realizations, [0, 1, 0, 1]) self.null_percentiles_expected_mean = np.mean(self.percentiles.data) self.null_percentiles_expected = np.array([ np.full((3, 3), 10.265101), np.full((3, 3), 10.4), np.full((3, 3), 10.534898), ])
def test_standard_grid_metadata(self): """Test standard grid metadata""" result = set_up_percentile_cube(self.data, self.percentiles, standard_grid_metadata="uk_ens") self.assertEqual(result.attributes["mosg__grid_type"], "standard") self.assertEqual(result.attributes["mosg__grid_version"], "1.3.0") self.assertEqual(result.attributes["mosg__grid_domain"], "uk_extended") self.assertEqual(result.attributes["mosg__model_configuration"], "uk_ens")
def test_error_non_probability_cube(self): """Test failure if cube doesn't contain probabilities""" perc_cube = set_up_percentile_cube( np.ones((3, 3, 3), dtype=np.float32), np.array((25, 50, 75), dtype=np.float32), ) plugin = OccurrenceBetweenThresholds([[25, 50]], "K") msg = "Input is not a probability cube" with self.assertRaisesRegex(ValueError, msg): plugin(perc_cube)
def setUp(self): """ Create a raw and calibrated cube with with forecast_reference_time and forecast_period coordinates. """ self.raw_cube = set_up_variable_cube(ECC_TEMPERATURE_REALIZATIONS) self.post_processed_percentiles = set_up_percentile_cube( np.sort(ECC_TEMPERATURE_REALIZATIONS, axis=0), np.array([10, 50, 90], dtype=np.float32), )
def setUp(self): """ Create a raw realization forecast with a fixed set of realization numbers and a percentile cube following ensemble reordering. """ self.raw_cube = set_up_variable_cube( ECC_TEMPERATURE_REALIZATIONS.copy(), realizations=[10, 11, 12]) self.post_processed_percentiles = set_up_percentile_cube( np.sort(ECC_TEMPERATURE_REALIZATIONS.copy(), axis=0), np.array([10, 50, 90], dtype=np.float32), )
def setUp(self): """Set up a percentile cube""" # Create a percentile cube land_data = np.ones((2, 3, 4), dtype=np.float32) sea_data = np.ones((2, 3, 4), dtype=np.float32) * 3.0 mask = np.array([ [ [True, False, False, False], [True, False, False, False], [False, False, False, True], ], [ [True, False, False, False], [True, False, False, False], [False, False, False, True], ], ]) land_data = np.ma.MaskedArray(land_data, mask) self.percentiles_land = set_up_percentile_cube(land_data, [30, 60]) self.percentiles_sea = set_up_percentile_cube(sea_data, [30, 60])
def test_ordering_for_percentile_coordinate(self): """Test that the cube has been reordered, if it is originally in an undesirable order and the cube contains a "percentile" coordinate.""" data = np.ones((3, 4, 5), dtype=np.float32) cube = set_up_percentile_cube(data, np.array([10, 50, 90])) cube.transpose([2, 1, 0]) save_netcdf(cube, self.filepath) result = load_cube(self.filepath) self.assertEqual(result.coord_dims("percentile")[0], 0) self.assertArrayAlmostEqual(result.coord_dims("latitude")[0], 1) self.assertArrayAlmostEqual(result.coord_dims("longitude")[0], 2)
def setUp(self): """Set up percentile cube.""" data = np.tile(np.linspace(5, 10, 9), 3).reshape(3, 3, 3) data[0] -= 1 data[1] += 1 data[2] += 3 self.percentile_cube = set_up_percentile_cube( data.astype(np.float32), np.array([10, 50, 90], dtype=np.float32), name="air_temperature", units="degC", )
def test_check_data_multiple_timesteps(self): """ Test that the plugin returns an Iris.cube.Cube with the expected data values for the percentiles. """ expected = np.array( [ [ [[4.5, 5.21428571], [5.92857143, 6.64285714]], [[7.35714286, 8.07142857], [8.78571429, 9.5]], ], [ [[6.5, 7.21428571], [7.92857143, 8.64285714]], [[9.35714286, 10.07142857], [10.78571429, 11.5]], ], [ [[7.5, 8.21428571], [8.92857143, 9.64285714]], [[10.35714286, 11.07142857], [11.78571429, 12.5]], ], ] ) cube = set_up_percentile_cube( np.zeros((3, 2, 2), dtype=np.float32), self.percentiles, name="air_temperature", units="degC", time=datetime(2015, 11, 23, 7), frt=datetime(2015, 11, 23, 6), ) cube = add_coordinate( cube, [datetime(2015, 11, 23, 7), datetime(2015, 11, 23, 8)], "time", is_datetime=True, order=[1, 0, 2, 3], ) data = np.tile(np.linspace(5, 10, 8), 3).reshape(3, 2, 2, 2) data[0] -= 1 data[1] += 1 data[2] += 3 cube.data = data.astype(np.float32) percentiles = [20, 60, 80] result = Plugin()._interpolate_percentiles( cube, percentiles, self.bounds_pairing, self.perc_coord ) self.assertArrayAlmostEqual(result.data, expected)
def setUp(self): """Set up percentile cube.""" data = np.tile(np.linspace(5, 10, 9), 3).reshape(3, 3, 3) data[0] -= 1 data[1] += 1 data[2] += 3 self.perc_coord = "percentile" self.percentiles = np.array([10, 50, 90], dtype=np.float32) self.cube = set_up_percentile_cube( np.sort(data.astype(np.float32), axis=0), self.percentiles, name="air_temperature", units="degC", ) self.bounds_pairing = (-40, 50)
def setUp(self): """Set up percentile cube.""" data = np.tile(np.linspace(5, 10, 9), 3).reshape(3, 3, 3) data[0] -= 1 data[1] += 1 data[2] += 3 self.percentile_cube = set_up_percentile_cube( data.astype(np.float32), np.array([10, 50, 90], dtype=np.float32), name="air_temperature", units="degC", ) self.expected = np.array([ [[4.75, 5.375, 6.0], [6.625, 7.25, 7.875], [8.5, 9.125, 9.75]], [[6.0, 6.625, 7.25], [7.875, 8.5, 9.125], [9.75, 10.375, 11.0]], [[7.25, 7.875, 8.5], [9.125, 9.75, 10.375], [11.0, 11.625, 12.25]], ])
def error_percentile_cube(error_thresholds): """Create sample error-percentile cube""" data = np.broadcast_to(error_thresholds[1:-1, np.newaxis, np.newaxis], (4, 10, 10)) percentile_cube = set_up_percentile_cube( data.astype(np.float32), percentiles=np.array([20.0, 40.0, 60.0, 80.0], dtype=np.float32), name="forecast_error_of_lwe_thickness_of_precipitation_amount", units="m", attributes=ATTRIBUTES, ) error_percentile_cube = add_coordinate( percentile_cube, coord_points=np.arange(5), coord_name="realization", coord_units="1", order=[0, 1, 2, 3], ) return error_percentile_cube
def percentile_cube(frt_points, time, frt): """Create a percentile cube for testing.""" cube = set_up_percentile_cube( np.zeros((6, 2, 2), dtype=np.float32), np.arange(0, 101, 20).astype(np.float32), name="air_temperature", units="C", time=time, frt=frt, ) cube = add_coordinate( cube, frt_points, "forecast_reference_time", is_datetime=True, order=(1, 0, 2, 3), ) cube.data = np.reshape(PERCENTILE_DATA, (6, 3, 2, 2)).astype(np.float32) return cube
def test_simple_check_data(self): """ Test that the plugin returns an Iris.cube.Cube with the expected forecast values at each percentile. """ data = np.array([8, 10, 12]) data = data[:, np.newaxis, np.newaxis] expected = data.copy() cube = set_up_percentile_cube( data.astype(np.float32), self.percentiles, name="air_temperature", units="degC", ) result = Plugin()._interpolate_percentiles(cube, self.percentiles, self.bounds_pairing, self.perc_coord) self.assertArrayAlmostEqual(result.data, expected)
def setUp(self): """ Create a cube with a realization coordinate and a cube with a percentile coordinate with forecast_reference_time and forecast_period coordinates. """ data = np.tile(np.linspace(5, 10, 9), 3).reshape(3, 3, 3) data[0] -= 1 data[1] += 1 data[2] += 3 self.realization_cube = set_up_variable_cube(data.astype(np.float32), name="air_temperature", units="degC") self.percentile_cube = set_up_percentile_cube( np.sort(data.astype(np.float32), axis=0), np.array([10, 50, 90], dtype=np.float32), name="air_temperature", units="degC", ) self.perc_coord = "percentile"
def percentile_fixture(): """Percentiles of wind gust from MOGREPS-UK""" data = np.array([[[2, 4], [4, 2]], [[5, 8], [6, 6]], [[12, 16], [16, 15]]], dtype=np.float32) percentiles = np.array([10, 50, 90], dtype=np.float32) attributes = { "source": "Met Office Unified Model", "title": "MOGREPS-UK Model Forecast on 2 km Standard Grid", "institution": "Met Office", "mosg__model_configuration": "uk_ens", "wind_gust_diagnostic": "Typical gusts", } return set_up_percentile_cube( data, percentiles, name="wind_gust", units="m s-1", attributes=attributes, spatial_grid="equalarea", )
def setUp(self): """Set up a test cube with the following data and coordinates, which comply with the IMPROVER datatypes and units standard (Units in parentheses are not mandatory): +-------------------------+-------------+----------------------+ | Name | Datatype | Units +=========================+=============+======================+ | data (air_temperature) | np.float32 | (Kelvin) | +-------------------------+-------------+----------------------+ | projection_x_coordinate | np.float32 | (metres) | +-------------------------+-------------+----------------------+ | projection_y_coordinate | np.float32 | (metres) | +-------------------------+-------------+----------------------+ | time | np.int64 | seconds since 1970.. | +-------------------------+-------------+----------------------+ | forecast_reference_time | np.int64 | seconds since 1970.. | +-------------------------+-------------+----------------------+ | forecast_period | np.int32 | seconds | +-------------------------+-------------+----------------------+ """ self.cube = set_up_variable_cube( 280 * np.ones((3, 3), dtype=np.float32), spatial_grid="equalarea" ) data = np.ones((3, 3, 3), dtype=np.float32) thresholds = np.array([272, 273, 274], dtype=np.float32) self.probability_cube = set_up_probability_cube(data, thresholds) data = np.array( [ 274 * np.ones((3, 3), dtype=np.float32), 275 * np.ones((3, 3), dtype=np.float32), 276 * np.ones((3, 3), dtype=np.float32), ] ) percentiles = np.array([25, 50, 75], np.float32) self.percentile_cube = set_up_percentile_cube(data, percentiles)
def setUp(self): """Set up temperature percentile cube for testing""" self.cube = set_up_percentile_cube( np.sort(ECC_TEMPERATURE_REALIZATIONS, axis=0), np.array([10, 50, 90], dtype=np.float32), )
def generate_metadata( name="air_pressure_at_sea_level", units=None, time_period=None, ensemble_members=8, leading_dimension=None, cube_type="variable", spp__relative_to_threshold="greater_than", npoints=71, **kwargs, ): """ Generate a cube with metadata only. Args: name (str): Output variable name, or if creating a probability cube the name of the underlying variable to which the probability field applies. units (Optional[str]): Output variable units, or if creating a probability cube the units of the underlying variable / threshold. time_period (Optional[int]): The period in minutes between the time bounds. This is used to calculate the lower time bound. If unset the diagnostic will be instantaneous, i.e. without time bounds. ensemble_members (Optional[int]): Number of ensemble members. Default 8, unless percentile or probability set to True. leading_dimension (Optional[List[float]]): List of realizations, percentiles or thresholds. cube_type (Optional[str]): The type of cube to be generated. Permitted values are "variable", "percentile" or "probability". spp__relative_to_threshold (Optional[str]): Value of the attribute "spp__relative_to_threshold" which is required for IMPROVER probability cubes. npoints (Optional[int]): Number of points along each of the y and x spatial axes. **kwargs: Additional keyword arguments to pass to the required cube setup function. Returns: iris.cube.Cube: Output of set_up_variable_cube(), set_up_percentile_cube() or set_up_probability_cube() """ if cube_type not in CUBE_TYPES: raise ValueError( 'Cube type {} not supported. Specify one of "variable", "percentile" or "probability".' .format(cube_type)) if "spatial_grid" in kwargs and kwargs["spatial_grid"] not in ( "latlon", "equalarea", ): raise ValueError( "Spatial grid {} not supported. Specify either latlon or equalarea." .format(kwargs["spatial_grid"])) if ("domain_corner" in kwargs and kwargs["domain_corner"] is not None and len(kwargs["domain_corner"]) != 2): raise ValueError("Domain corner must be a list or tuple of length 2.") if units is None: units = _get_units(name) # If time_period specified, create time bounds using time as upper bound if time_period is not None: if "time" not in kwargs: kwargs["time"] = DEFAULT_TIME time_bounds = _create_time_bounds(kwargs["time"], time_period) kwargs["time_bounds"] = time_bounds # If grid_spacing not specified, use default for requested spatial grid if "grid_spacing" not in kwargs or kwargs["grid_spacing"] is None: if "spatial_grid" not in kwargs: kwargs["spatial_grid"] = DEFAULT_SPATIAL_GRID kwargs["grid_spacing"] = DEFAULT_GRID_SPACING[kwargs["spatial_grid"]] # Create ndimensional array of zeros if "height_levels" not in kwargs: kwargs["height_levels"] = None data = _create_data_array(ensemble_members, leading_dimension, npoints, kwargs["height_levels"]) # Set up requested cube if cube_type == "percentile": metadata_cube = set_up_percentile_cube( data, percentiles=leading_dimension, name=name, units=units, **kwargs, ) elif cube_type == "probability": metadata_cube = set_up_probability_cube( data, leading_dimension, variable_name=name, threshold_units=units, spp__relative_to_threshold=spp__relative_to_threshold, **kwargs, ) else: metadata_cube = set_up_variable_cube( data, name=name, units=units, realizations=leading_dimension, **kwargs, ) metadata_cube = squeeze(metadata_cube) return metadata_cube
def test_single_percentile(self): """Test a cube with one percentile correctly stores this as a scalar coordinate""" result = set_up_percentile_cube(self.data[1:2], self.percentiles[1:2]) dim_coords = get_dim_coord_names(result) self.assertNotIn("percentile", dim_coords)
def setUp(self): """Set-up cubes for testing.""" thresholds = [283, 288] percentiles = [25, 75] probability_data = np.ones((2, 4, 4), dtype=np.float32) realization_data = np.ones((4, 4), dtype=np.float32) self.truth_attribute = "mosg__model_configuration=uk_det" # Set-up probability and realization forecast cubes self.probability_forecast = CubeList( [set_up_probability_cube(probability_data, thresholds)] ) self.realization_forecast = CubeList([set_up_variable_cube(realization_data)]) self.percentile_forecast = CubeList( [set_up_percentile_cube(probability_data, percentiles)] ) # Set-up coefficient cubes fp_names = [self.realization_forecast[0].name()] predictor_index = iris.coords.DimCoord( np.array(range(len(fp_names)), dtype=np.int32), long_name="predictor_index", units="1", ) dim_coords_and_dims = ((predictor_index, 0),) predictor_name = iris.coords.AuxCoord( fp_names, long_name="predictor_name", units="no_unit" ) aux_coords_and_dims = ((predictor_name, 0),) attributes = { "diagnostic_standard_name": self.realization_forecast[0].name(), "distribution": "norm", } alpha = iris.cube.Cube( np.array(0, dtype=np.float32), long_name="emos_coefficients_alpha", units="K", attributes=attributes, ) beta = iris.cube.Cube( np.array([0.5], dtype=np.float32), long_name="emos_coefficients_beta", units="1", attributes=attributes, dim_coords_and_dims=dim_coords_and_dims, aux_coords_and_dims=aux_coords_and_dims, ) gamma = iris.cube.Cube( np.array(0, dtype=np.float32), long_name="emos_coefficients_gamma", units="K", attributes=attributes, ) delta = iris.cube.Cube( np.array(1, dtype=np.float32), long_name="emos_coefficients_delta", units="1", attributes=attributes, ) self.coefficient_cubelist = CubeList([alpha, beta, gamma, delta]) # Set-up land-sea mask. self.land_sea_mask_name = "land_binary_mask" self.land_sea_mask = set_up_variable_cube( np.zeros((4, 4), dtype=np.float32), name=self.land_sea_mask_name ) for coord in ["time", "forecast_reference_time", "forecast_period"]: self.land_sea_mask.remove_coord(coord) self.land_sea_mask = CubeList([self.land_sea_mask]) altitude = set_up_variable_cube( np.ones((4, 4), dtype=np.float32), name="surface_altitude", units="m" ) for coord in ["time", "forecast_reference_time", "forecast_period"]: altitude.remove_coord(coord) self.additional_predictors = CubeList([altitude])
def setUp(self): """Create a wind-speed and wind-gust cube with percentile coord.""" data = np.zeros((2, 3, 3), dtype=np.float32) percentiles = np.array([50.0, 90.0], dtype=np.float32) self.cube_wg = set_up_percentile_cube(data, percentiles)
def setUp(self): """Set up some "uncalibrated forecast" inputs""" attributes = { "title": "MOGREPS-UK Forecast", "source": "Met Office Unified Model", "institution": "Met Office", } forecast = np.array( [ np.full((3, 3), 10.4), np.full((3, 3), 10.8), np.full((3, 3), 10.1) ], dtype=np.float32, ) self.realizations = set_up_variable_cube(forecast, units="degC", attributes=attributes) percentiles = np.array( [ np.full((3, 3), 10.2), np.full((3, 3), 10.4), np.full((3, 3), 10.6) ], dtype=np.float32, ) self.percentiles = set_up_percentile_cube( percentiles, np.array([25, 50, 75], dtype=np.float32), units="degC", attributes=attributes, ) probabilities = np.array( [np.full((3, 3), 1), np.full((3, 3), 0.9), np.full((3, 3), 0)], dtype=np.float32, ) self.probabilities = set_up_probability_cube( probabilities, np.array([9, 10, 11], dtype=np.float32), threshold_units="degC", attributes=attributes, ) self.coefficients = build_coefficients_cubelist( self.realizations, [0, 1, 0, 1], CubeList([self.realizations])) self.null_percentiles_expected_mean = np.mean(self.percentiles.data) self.null_percentiles_expected = np.array([ np.full((3, 3), 10.265101), np.full((3, 3), 10.4), np.full((3, 3), 10.534898), ]) self.alternative_percentiles = [25.0, 50.0, 75.0] land_sea_data = np.array([[1, 1, 0], [1, 1, 0], [1, 0, 0]], dtype=np.int32) self.land_sea_mask = set_up_variable_cube(land_sea_data, name="land_binary_mask", units="1") # Generate site forecast and additional predictor cubes. data = np.tile([1.6, 1.3, 1.4, 1.1], (4, 1)) altitude = np.array([10, 20, 30, 40]) latitude = np.linspace(58.0, 59.5, 4) longitude = np.linspace(-0.25, 0.5, 4) wmo_id = ["03001", "03002", "03003", "03004"] time_coords = construct_scalar_time_coords( datetime.datetime(2017, 11, 5, 4, 0), None, datetime.datetime(2017, 11, 5, 0, 0), ) time_coords = [t[0] for t in time_coords] realization_coord = [ iris.coords.DimCoord(np.arange(1, 5), standard_name="realization") ] self.realizations_spot_cube = build_spotdata_cube( data, "air_temperature", "degC", altitude, latitude, longitude, wmo_id, scalar_coords=time_coords, additional_dims=realization_coord, ) self.realizations_spot_cube.attributes.update( MANDATORY_ATTRIBUTE_DEFAULTS) self.spot_altitude_cube = self.realizations_spot_cube[0].copy( self.realizations_spot_cube.coord("altitude").points) self.spot_altitude_cube.rename("altitude") self.spot_altitude_cube.units = "m" for coord in [ "altitude", "forecast_period", "forecast_reference_time", "realization", "time", ]: self.spot_altitude_cube.remove_coord(coord) self.spot_coefficients = build_coefficients_cubelist( self.realizations_spot_cube, [0, [0.9, 0.1], 0, 1], CubeList([self.realizations_spot_cube, self.spot_altitude_cube]), )
def percentile_cube(): data = np.array([range( 10, 19), range(20, 29), range(40, 49)], dtype=np.float32).reshape([3, 3, 3]) return set_up_percentile_cube(data, percentiles=[25, 50, 75])
def percentile_cube(): """Fixture for is_percentile tests""" data = np.zeros((2, 3, 3), dtype=np.float32) percentiles = np.array([50.0, 90.0], dtype=np.float32) return set_up_percentile_cube(data, percentiles)