def shape_weights(cube: Cube, weights: Cube) -> ndarray: """ The function shapes weights to match the diagnostic cube. A cube of weights that vary across the blending coordinate will be broadcast to match the complete multidimensional cube shape. A multidimensional cube of weights will be checked to ensure that the coordinate names match between the two cubes. If they match the order will be enforced and then the shape will be checked. If the shapes match the weights will be returned as an array. Args: cube: The data cube on which a coordinate is being blended. weights: Cube of blending weights. Returns: An array of weights that matches the cube data shape. Raises: ValueError: If weights cube coordinates do not match the diagnostic cube in the case of a multidimensional weights cube. ValueError: If weights cube shape is not broadcastable to the data cube shape. """ # Check that a multidimensional weights cube has coordinates that match # the diagnostic cube. Checking names only to not to be too exacting. weight_dims = get_dim_coord_names(weights) cube_dims = get_dim_coord_names(cube) if set(weight_dims) == set(cube_dims): enforce_coordinate_ordering(weights, cube_dims) weights_array = weights.data.astype(FLOAT_DTYPE) else: # Map array of weights to shape of cube to collapse. dim_map = [] dim_coords = [coord.name() for coord in weights.dim_coords] # Loop through dim coords in weights cube and find the dim the # coord relates to in the cube we are collapsing. for dim_coord in dim_coords: try: dim_map.append(cube.coord_dims(dim_coord)[0]) except CoordinateNotFoundError: message = ( "{} is a coordinate on the weights cube but it is not " "found on the cube we are trying to collapse.") raise ValueError(message.format(dim_coord)) try: weights_array = iris.util.broadcast_to_shape( np.array(weights.data, dtype=FLOAT_DTYPE), cube.shape, tuple(dim_map), ) except ValueError: msg = ("Weights cube is not a compatible shape with the" " data cube. Weights: {}, Diagnostic: {}".format( weights.shape, cube.shape)) raise ValueError(msg) return weights_array
def test_preserves_dimension_order(self): """Test order of original cube dimensions is preserved on subsetting""" self.uk_cube.transpose([1, 0]) expected_dims = get_dim_coord_names(self.uk_cube) result = subset_data(self.uk_cube, grid_spec=self.grid_spec) result_dims = get_dim_coord_names(result) self.assertSequenceEqual(result_dims, expected_dims)
def _create_template_slice(self, cube_to_collapse): """ Create a template cube from a slice of the cube we are collapsing. The slice will be over blend_coord, y and x and will remove any other dimensions. This means that the resulting spatial weights won't vary in any other dimension other than the blend_coord. If the mask does vary in another dimension an error is raised. Args: cube_to_collapse (iris.cube.Cube): The cube that will be collapsed along the blend_coord using the spatial weights generated using this plugin. Must be masked where there is invalid data. Returns: iris.cube.Cube: A cube with dimensions blend_coord, y, x, on which to shape the output weights cube. Raises: ValueError: if the blend coordinate is associated with more than one dimension on the cube to collapse, or no dimension ValueError: if the mask on cube_to_collapse varies along a dimension other than the dimension associated with blend_coord. """ self.blend_coord = find_blend_dim_coord(cube_to_collapse, self.blend_coord) # Find original dim coords in input cube original_dim_coords = get_dim_coord_names(cube_to_collapse) # Slice over required coords x_coord = cube_to_collapse.coord(axis="x").name() y_coord = cube_to_collapse.coord(axis="y").name() coords_to_slice_over = [self.blend_coord, y_coord, x_coord] if original_dim_coords == coords_to_slice_over: return cube_to_collapse # Check mask does not vary over additional dimensions slices = cube_to_collapse.slices(coords_to_slice_over) first_slice = next(slices) if np.ma.is_masked(first_slice.data): first_mask = first_slice.data.mask for cube_slice in slices: if not np.all(cube_slice.data.mask == first_mask): message = ( "The mask on the input cube can only vary along the " "blend_coord, differences in the mask were found " "along another dimension") raise ValueError(message) # Remove non-spatial non-blend dimensions, returning a 3D template cube without # additional scalar coordinates (eg realization) for coord in original_dim_coords: if coord not in coords_to_slice_over: first_slice.remove_coord(coord) # Return slice template return first_slice
def test_single_percentile(self): """Test dimensions of output at median only""" collapse_coord = ["realization"] plugin = PercentileConverter(collapse_coord, percentiles=[50]) result = plugin.process(self.cube) result_coords = get_coord_names(result) self.assertNotIn("realization", result_coords) self.assertIn("percentile", result_coords) self.assertNotIn("percentile", get_dim_coord_names(result))
def _create_output_cube(self, template, data, points, bounds): """ Populates a template cube with data from the integration Args: template (iris.cube.Cube): Copy of upper or lower bounds cube, based on direction of integration data (list or numpy.ndarray): Integrated data points (list or numpy.ndarray): Points values for the integrated coordinate. These will not match the template cube if any slices were skipped in the integration, and therefore are used to slice the template cube to match the data array. bounds (list or numpy.ndarray): Bounds values for the integrated coordinate Returns: iris.cube.Cube """ # extract required slices from template cube template = template.extract( iris.Constraint(coord_values={ self.coord_name_to_integrate: lambda x: x in points })) # re-promote integrated coord to dimension coord if need be aux_coord_names = [coord.name() for coord in template.aux_coords] if self.coord_name_to_integrate in aux_coord_names: template = iris.util.new_axis(template, self.coord_name_to_integrate) # order dimensions on the template cube so that the integrated # coordinate is first (as this is the leading dimension on the # data array) enforce_coordinate_ordering(template, self.coord_name_to_integrate) # generate appropriate metadata for new cube attributes = generate_mandatory_attributes([template]) coord_dtype = template.coord(self.coord_name_to_integrate).dtype name, units = self._generate_output_name_and_units() # create new cube from template integrated_cube = create_new_diagnostic_cube(name, units, template, attributes, data=np.array(data)) integrated_cube.coord(self.coord_name_to_integrate).bounds = np.array( bounds).astype(coord_dtype) # re-order cube to match dimensions of input cube ordered_dimensions = get_dim_coord_names(self.input_cube) enforce_coordinate_ordering(integrated_cube, ordered_dimensions) return integrated_cube
def test_null_percentiles(self): """Test effect of "neutral" emos coefficients in percentile space (this is small but non-zero due to limited sampling of the distribution)""" result = ApplyEMOS()(self.percentiles, self.coefficients, realizations_count=3) self.assertIn("percentile", get_dim_coord_names(result)) self.assertArrayAlmostEqual(result.data, self.null_percentiles_expected) self.assertAlmostEqual( np.mean(result.data), self.null_percentiles_expected_mean )
def remove_cube_halo(cube, halo_radius): """ Remove halo of halo_radius from a cube. This function converts the halo radius into the number of grid points in the x and y coordinate that need to be removed. It then calls remove_halo_from_cube which only acts on a cube with x and y coordinates so we need to slice the cube and them merge the cube back together ensuring the resulting cube has the same dimension coordinates. Args: cube (iris.cube.Cube): Cube on extended grid halo_radius (float): Size of border to remove, in metres Returns: iris.cube.Cube: New cube with the halo removed. """ halo_size_x = distance_to_number_of_grid_cells(cube, halo_radius, axis="x") halo_size_y = distance_to_number_of_grid_cells(cube, halo_radius, axis="y") result_slices = iris.cube.CubeList() for cube_slice in cube.slices([cube.coord(axis="y"), cube.coord(axis="x")]): cube_halo = remove_halo_from_cube(cube_slice, halo_size_x, halo_size_y) result_slices.append(cube_halo) result = result_slices.merge_cube() # re-promote any scalar dimensions lost in slice / merge req_dims = get_dim_coord_names(cube) present_dims = get_dim_coord_names(result) for coord in req_dims: if coord not in present_dims: result = iris.util.new_axis(result, coord) # re-order (needed if scalar dimensions have been re-added) enforce_coordinate_ordering(result, req_dims) return result
def test_null_realizations(self): """Test effect of "neutral" emos coefficients in realization space""" expected_mean = np.mean(self.realizations.data) expected_data = np.array([ np.full((3, 3), 10.433333), np.full((3, 3), 10.670206), np.full((3, 3), 10.196461), ]) result = ApplyEMOS()(self.realizations, self.coefficients) self.assertIn("realization", get_dim_coord_names(result)) self.assertArrayAlmostEqual(result.data, expected_data) self.assertAlmostEqual(np.mean(result.data), expected_mean)
def test_null_percentiles_truncnorm_standard_shape_parameters(self): """Test effect of "neutral" emos coefficients in percentile space (this is small but non-zero due to limited sampling of the distribution) for the truncated normal distribution.""" coefficients = iris.cube.CubeList([]) for cube in self.coefficients: cube.attributes["distribution"] = "truncnorm" cube.attributes["shape_parameters"] = np.array([0, np.inf], np.float32) coefficients.append(cube) result = ApplyEMOS()(self.percentiles, coefficients, realizations_count=3) self.assertIn("percentile", get_dim_coord_names(result)) self.assertArrayAlmostEqual(result.data, self.null_percentiles_expected) self.assertAlmostEqual( np.mean(result.data), self.null_percentiles_expected_mean )
def test_null_percentiles(self): """Test effect of "neutral" emos coefficients in percentile space (this is small but non-zero due to limited sampling of the distribution)""" expected_mean = np.mean(self.percentiles.data) expected_data = np.array([ np.full((3, 3), 10.265101), np.full((3, 3), 10.4), np.full((3, 3), 10.534898), ]) result = ApplyEMOS()(self.percentiles, self.coefficients, realizations_count=3) self.assertIn("percentile", get_dim_coord_names(result)) self.assertArrayAlmostEqual(result.data, expected_data) self.assertAlmostEqual(np.mean(result.data), expected_mean)
def thin_cube(cube: Cube, thinning_dict: Dict[str, int]) -> Cube: """ Thin the coordinate by taking every X points, defined in the thinning dict as {coordinate: X} Args: cube: The cube containing the coordinates to be thinned. thinning_dict: A dictionary of coordinate and the step value, i.e. a step of 2 will skip every other point Returns: A cube with thinned coordinates. """ coord_names = get_dim_coord_names(cube) slices = [slice(None, None, None)] * len(coord_names) for key, val in thinning_dict.items(): slices[coord_names.index(key)] = slice(None, None, val) return cube[tuple(slices)]
def test_null_percentiles_truncnorm_alternative_shape_parameters(self): """Test effect of "neutral" emos coefficients in percentile space (this is small but non-zero due to limited sampling of the distribution) for the truncated normal distribution with alternative shape parameters to show the truncnorm distribution having an effect.""" coefficients = iris.cube.CubeList([]) for cube in self.coefficients: cube.attributes["distribution"] = "truncnorm" cube.attributes["shape_parameters"] = np.array([10, np.inf], np.float32) coefficients.append(cube) expected_mean = np.mean(self.percentiles.data) expected_data = np.array( [ np.full((3, 3), 10.275656), np.full((3, 3), 10.405704), np.full((3, 3), 10.5385), ] ) result = ApplyEMOS()(self.percentiles, coefficients, realizations_count=3) self.assertIn("percentile", get_dim_coord_names(result)) self.assertArrayAlmostEqual(result.data, expected_data) self.assertNotAlmostEqual(np.mean(result.data), expected_mean)
def process(self, cube, weights=None): """Calculate weighted blend across the chosen coord, for either probabilistic or percentile data. If there is a percentile coordinate on the cube, it will blend using the PercentileBlendingAggregator but the percentile coordinate must have at least two points. Args: cube (iris.cube.Cube): Cube to blend across the coord. weights (iris.cube.Cube): Cube of blending weights. This will have 1 or 3 dimensions, corresponding either to blend dimension on the input cube with or without and additional 2 spatial dimensions. If None, the input cube is blended with equal weights across the blending dimension. Returns: iris.cube.Cube: Containing the weighted blend across the chosen coordinate (typically forecast reference time or model). Raises: TypeError : If the first argument not a cube. CoordinateNotFoundError : If coordinate to be collapsed not found in cube. CoordinateNotFoundError : If coordinate to be collapsed not found in provided weights cube. ValueError : If coordinate to be collapsed is not a dimension. """ if not isinstance(cube, iris.cube.Cube): msg = ("The first argument must be an instance of iris.cube.Cube " "but is {}.".format(type(cube))) raise TypeError(msg) if not cube.coords(self.blend_coord): msg = "Coordinate to be collapsed not found in cube." raise CoordinateNotFoundError(msg) output_dims = get_dim_coord_names( next(cube.slices_over(self.blend_coord))) self.blend_coord = find_blend_dim_coord(cube, self.blend_coord) # Ensure input cube and weights cube are ordered equivalently along # blending coordinate. cube = sort_coord_in_cube(cube, self.blend_coord) if weights is not None: if not weights.coords(self.blend_coord): msg = "Coordinate to be collapsed not found in weights cube." raise CoordinateNotFoundError(msg) weights = sort_coord_in_cube(weights, self.blend_coord) # Check that the time coordinate is single valued if required. self.check_compatible_time_points(cube) # Do blending and update metadata if self.check_percentile_coord(cube): enforce_coordinate_ordering(cube, [self.blend_coord, "percentile"]) result = self.percentile_weighted_mean(cube, weights) else: enforce_coordinate_ordering(cube, [self.blend_coord]) result = self.weighted_mean(cube, weights) # Reorder resulting dimensions to match input enforce_coordinate_ordering(result, output_dims) return result
def process(self, temperature, orography, land_sea_mask, model_id_attr=None): """Calculates the lapse rate from the temperature and orography cubes. Args: temperature (iris.cube.Cube): Cube of air temperatures (K). orography (iris.cube.Cube): Cube containing orography data (metres) land_sea_mask (iris.cube.Cube): Cube containing a binary land-sea mask. True for land-points and False for Sea. model_id_attr (str): Name of the attribute used to identify the source model for blending. This is inherited from the input temperature cube. Returns: iris.cube.Cube: Cube containing lapse rate (K m-1) Raises ------ TypeError: If input cubes are not cubes ValueError: If input cubes are the wrong units. """ if not isinstance(temperature, iris.cube.Cube): msg = "Temperature input is not a cube, but {}" raise TypeError(msg.format(type(temperature))) if not isinstance(orography, iris.cube.Cube): msg = "Orography input is not a cube, but {}" raise TypeError(msg.format(type(orography))) if not isinstance(land_sea_mask, iris.cube.Cube): msg = "Land/Sea mask input is not a cube, but {}" raise TypeError(msg.format(type(land_sea_mask))) # Converts cube units. temperature_cube = temperature.copy() temperature_cube.convert_units("K") orography.convert_units("metres") # Extract x/y co-ordinates. x_coord = temperature_cube.coord(axis="x").name() y_coord = temperature_cube.coord(axis="y").name() # Extract orography and land/sea mask data. orography_data = next(orography.slices([y_coord, x_coord])).data land_sea_mask_data = next(land_sea_mask.slices([y_coord, x_coord])).data # Fill sea points with NaN values. orography_data = np.where(land_sea_mask_data, orography_data, np.nan) # Create list of arrays over "realization" coordinate has_realization_dimension = False original_dimension_order = None if temperature_cube.coords("realization", dim_coords=True): original_dimension_order = get_dim_coord_names(temperature_cube) enforce_coordinate_ordering(temperature_cube, "realization") temp_data_slices = temperature_cube.data has_realization_dimension = True else: temp_data_slices = [temperature_cube.data] # Calculate lapse rate for each realization lapse_rate_data = [] for temperature_data in temp_data_slices: lapse_rate_array = self._generate_lapse_rate_array( temperature_data, orography_data, land_sea_mask_data) lapse_rate_data.append(lapse_rate_array) lapse_rate_data = np.array(lapse_rate_data) if not has_realization_dimension: lapse_rate_data = np.squeeze(lapse_rate_data) attributes = generate_mandatory_attributes([temperature], model_id_attr=model_id_attr) lapse_rate_cube = create_new_diagnostic_cube( "air_temperature_lapse_rate", "K m-1", temperature_cube, attributes, data=lapse_rate_data, ) if original_dimension_order: enforce_coordinate_ordering(lapse_rate_cube, original_dimension_order) return lapse_rate_cube
def test_single_threshold(self): """Test a cube with one threshold correctly stores this as a scalar coordinate""" result = set_up_probability_cube(self.data[1:2], self.thresholds[1:2]) dim_coords = get_dim_coord_names(result) self.assertNotIn("air_temperature", dim_coords)
def test_single_percentile(self): """Test a cube with one percentile correctly stores this as a scalar coordinate""" result = set_up_percentile_cube(self.data[1:2], self.percentiles[1:2]) dim_coords = get_dim_coord_names(result) self.assertNotIn("percentile", dim_coords)
def subset_data( cube: Cube, grid_spec: Optional[Dict[str, Dict[str, int]]] = None, site_list: Optional[List] = None, ) -> Cube: """Extract a spatial cutout or subset of sites from data to generate suite reference outputs. Args: cube: Input dataset grid_spec: Dictionary containing bounding grid points and an integer "thinning factor" for each of UK and global grid, to create cutouts. Eg a "thinning factor" of 10 would mean every 10th point being taken for the cutout. The expected dictionary has keys that are spatial coordinate names, with values that are dictionaries with "min", "max" and "thin" keys. site_list: List of WMO site IDs to extract. These IDs must match the type and format of the "wmo_id" coordinate on the input spot cube. Returns: Subset of input cube as specified by input constraints Raises: ValueError: If site_list is not provided for a spot data cube ValueError: If the spot data cube does not contain any of the required sites ValueError: If grid_spec is not provided for a gridded cube ValueError: If grid_spec does not contain entries for the spatial coordinates on the input gridded data ValueError: If the grid_spec provided does not overlap with the cube domain """ if cube.coords("spot_index"): if site_list is None: raise ValueError("site_list required to extract from spot data") constraint = Constraint( coord_values={"wmo_id": lambda x: x in site_list}) result = cube.extract(constraint) if result is None: raise ValueError( f"Cube does not contain any of the required sites: {site_list}" ) else: if grid_spec is None: raise ValueError("grid_spec required to extract from gridded data") x_coord = cube.coord(axis="x").name() y_coord = cube.coord(axis="y").name() for coord in [y_coord, x_coord]: if coord not in grid_spec: raise ValueError( f"Cube coordinates {y_coord}, {x_coord} are not present within " f"{grid_spec.keys()}") def _create_cutout(cube, grid_spec): """Given a gridded data cube and boundary limits for cutout dimensions, create cutout. Expects cube on either lat-lon or equal area grid. """ x_coord = cube.coord(axis="x").name() y_coord = cube.coord(axis="y").name() xmin = grid_spec[x_coord]["min"] xmax = grid_spec[x_coord]["max"] ymin = grid_spec[y_coord]["min"] ymax = grid_spec[y_coord]["max"] # need to use cube intersection for circular coordinates (longitude) if x_coord == "longitude": lat_constraint = Constraint( latitude=lambda y: ymin <= y.point <= ymax) cutout = cube.extract(lat_constraint) if cutout is None: return cutout cutout = cutout.intersection(longitude=(xmin, xmax), ignore_bounds=True) # intersection creates a new coordinate with default datatype - we # therefore need to re-cast to meet the IMPROVER standard cutout.coord("longitude").points = cutout.coord( "longitude").points.astype(FLOAT_DTYPE) if cutout.coord("longitude").bounds is not None: cutout.coord("longitude").bounds = cutout.coord( "longitude").bounds.astype(FLOAT_DTYPE) else: x_constraint = Constraint( projection_x_coordinate=lambda x: xmin <= x.point <= xmax) y_constraint = Constraint( projection_y_coordinate=lambda y: ymin <= y.point <= ymax) cutout = cube.extract(x_constraint & y_constraint) return cutout cutout = _create_cutout(cube, grid_spec) if cutout is None: raise ValueError( "Cube domain does not overlap with cutout specified:\n" f"{x_coord}: {grid_spec[x_coord]}, {y_coord}: {grid_spec[y_coord]}" ) original_coords = get_dim_coord_names(cutout) thin_x = grid_spec[x_coord]["thin"] thin_y = grid_spec[y_coord]["thin"] result_list = CubeList() try: for subcube in cutout.slices([y_coord, x_coord]): result_list.append(subcube[::thin_y, ::thin_x]) except ValueError as cause: # error is raised if X or Y coordinate are single-valued (non-dimensional) if "iterator" in str(cause) and "dimension" in str(cause): raise ValueError( "Function does not support single point extraction") else: raise result = result_list.merge_cube() enforce_coordinate_ordering(result, original_coords) return result