def process(self, cube, mask_cube): """ 1. Iterate over the chosen coordinate within the mask_cube and apply the mask at each iteration to the cube that is to be neighbourhood processed. 2. Concatenate the cubes from each iteration together to create a single cube. Args: cube (Iris.cube.Cube): Cube containing the array to which the square neighbourhood will be applied. mask_cube (Iris.cube.Cube): Cube containing the array to be used as a mask. Returns: concatenated_cube (Iris.cube.Cube): Cube containing the smoothed field after the square neighbourhood method has been applied when applying masking for each point along the coord_for_masking coordinate. The resulting cube is concatenated so that the dimension coordinates match the input cube. """ yname = cube.coord(axis='y').name() xname = cube.coord(axis='x').name() result_slices = iris.cube.CubeList([]) # Take 2D slices of the input cube for memory issues. for x_y_slice in cube.slices([yname, xname]): cube_slices = iris.cube.CubeList([]) # Apply each mask in in mask_cube to the 2D input slice. for cube_slice in mask_cube.slices_over(self.coord_for_masking): output_cube = NeighbourhoodProcessing( self.neighbourhood_method, self.radii, lead_times=self.lead_times, weighted_mode=self.weighted_mode, sum_or_fraction=self.sum_or_fraction, re_mask=self.re_mask ).process(x_y_slice, mask_cube=cube_slice) coord_object = cube_slice.coord(self.coord_for_masking).copy() output_cube.add_aux_coord(coord_object) output_cube = iris.util.new_axis( output_cube, self.coord_for_masking) cube_slices.append(output_cube) concatenated_cube = cube_slices.concatenate_cube() exception_coordinates = ( find_dimension_coordinate_mismatch( x_y_slice, concatenated_cube, two_way_mismatch=False)) concatenated_cube = check_cube_coordinates( x_y_slice, concatenated_cube, exception_coordinates=exception_coordinates) result_slices.append(concatenated_cube) result = result_slices.merge_cube() exception_coordinates = ( find_dimension_coordinate_mismatch( cube, result, two_way_mismatch=False)) result = check_cube_coordinates( cube, result, exception_coordinates=exception_coordinates) return result
def run(self, cube, radius): """ Method to apply a circular kernel to the data within the input cube in order to derive percentiles over the kernel. Args: cube : Iris.cube.Cube Cube containing array to apply processing to. radius : Float Radius in metres for use in specifying the number of grid cells used to create a circular neighbourhood. Returns: result : Iris.cube.Cube Cube containing the percentile fields. Has percentile as an added dimension. """ # Check that the cube has an equal area grid. check_if_grid_is_equal_area(cube) # Take data array and identify X and Y axes indices ranges_tuple = convert_distance_into_number_of_grid_cells( cube, radius, MAX_RADIUS_IN_GRID_CELLS) ranges_xy = np.array(ranges_tuple) kernel = circular_kernel(ranges_xy, ranges_tuple, weighted_mode=False) # Loop over each 2D slice to reduce memory demand and derive # percentiles on the kernel. Will return an extra dimension. pctcubelist = iris.cube.CubeList() for slice_2d in cube.slices(['projection_y_coordinate', 'projection_x_coordinate']): pctcubelist.append( self.pad_and_unpad_cube(slice_2d, kernel)) result = pctcubelist.merge_cube() exception_coordinates = ( find_dimension_coordinate_mismatch( cube, result, two_way_mismatch=False)) result = ( check_cube_coordinates( cube, result, exception_coordinates=exception_coordinates)) # Arrange cube, so that the coordinate order is: # realization, percentile, other coordinates. required_order = [] if result.coords("realization"): if result.coords("realization", dimensions=[]): result = iris.util.new_axis(result, "realization") required_order.append(result.coord_dims("realization")[0]) if result.coords("percentiles_over_neighbourhood"): required_order.append( result.coord_dims("percentiles_over_neighbourhood")[0]) other_coords = [] for coord in result.dim_coords: if coord.name() not in ["realization", "percentiles_over_neighbourhood"]: other_coords.append(result.coord_dims(coord.name())[0]) required_order.extend(other_coords) result.transpose(required_order) return result
def test_basic(self): """Test that the method returns the expected cube type with coords""" result = self.plugin.process( CubeList([self.fg_cube, self.ltng_cube, self.precip_cube])) self.assertIsInstance(result, Cube) # We expect the threshold coordinate to have been removed. self.assertCountEqual( find_dimension_coordinate_mismatch(result, self.precip_cube), ['threshold']) self.assertTrue(result.name() == 'probability_of_lightning')
def test_two_way_mismatch(self): """Test when finding a two-way mismatch, when the first and second cube contain different coordinates.""" first_cube = self.cube.copy() second_cube = next(self.cube.slices_over("realization")).copy() second_cube.remove_coord("realization") second_cube = add_coordinate(second_cube, [10, 20], "height", "m") result = find_dimension_coordinate_mismatch(first_cube, second_cube) self.assertIsInstance(result, list) self.assertListEqual(result, ["height", "realization"])
def test_two_way_mismatch(self): """Test when finding a two-way mismatch, when the first and second cube contain different coordinates.""" cube = set_up_cube() first_cube = cube.copy() first_cube.remove_coord("time") second_cube = cube.copy() second_cube.remove_coord("realization") result = find_dimension_coordinate_mismatch(first_cube, second_cube) self.assertIsInstance(result, list) self.assertListEqual(result, ["time", "realization"])
def test_mismatch_in_first_cube(self): """Test when finding a one-way mismatch, so that the second cube has a missing coordinate. This returns an empty list.""" cube = set_up_cube() first_cube = cube.copy() second_cube = cube.copy() second_cube.remove_coord("time") result = find_dimension_coordinate_mismatch( first_cube, second_cube, two_way_mismatch=False) self.assertIsInstance(result, list) self.assertFalse(result)
def test_mismatch_in_second_cube(self): """Test when finding a one-way mismatch, so that the first cube has a missing coordinate. This returns a list with the missing coordinate name.l""" cube = set_up_cube() first_cube = cube.copy() first_cube.remove_coord("time") second_cube = cube.copy() result = find_dimension_coordinate_mismatch( first_cube, second_cube, two_way_mismatch=False) self.assertIsInstance(result, list) self.assertListEqual(result, ["time"])
def test_mismatch_in_second_cube(self): """Test when finding a one-way mismatch, so that the first cube has a missing coordinate. This returns a list with the missing coordinate name.""" first_cube = next(self.cube.slices_over("realization")).copy() first_cube.remove_coord("realization") second_cube = self.cube.copy() result = find_dimension_coordinate_mismatch(first_cube, second_cube, two_way_mismatch=False) self.assertIsInstance(result, list) self.assertListEqual(result, ["realization"])
def process(self, cube: Cube) -> Cube: """ Method to apply a circular kernel to the data within the input cube in order to derive percentiles over the kernel. Args: cube: Cube containing array to apply processing to. Returns: Cube containing the percentile fields. Has percentile as an added dimension. """ super().process(cube) if np.ma.is_masked(cube.data): msg = ("The use of masked input cubes is not yet implemented in" " the GeneratePercentilesFromANeighbourhood plugin.") raise NotImplementedError(msg) # Check that the cube has an equal area grid. check_if_grid_is_equal_area(cube) # Take data array and identify X and Y axes indices grid_cell = distance_to_number_of_grid_cells(cube, self.radius) check_radius_against_distance(cube, self.radius) kernel = circular_kernel(grid_cell, weighted_mode=False) # Loop over each 2D slice to reduce memory demand and derive # percentiles on the kernel. Will return an extra dimension. pctcubelist = iris.cube.CubeList() for slice_2d in cube.slices( ["projection_y_coordinate", "projection_x_coordinate"]): pctcubelist.append(self.pad_and_unpad_cube(slice_2d, kernel)) result = pctcubelist.merge_cube() exception_coordinates = find_dimension_coordinate_mismatch( cube, result, two_way_mismatch=False) result = check_cube_coordinates( cube, result, exception_coordinates=exception_coordinates) # Arrange cube, so that the coordinate order is: # realization, percentile, other coordinates. required_order = [] if result.coords("realization", dim_coords=True): required_order.append(result.coord_dims("realization")[0]) if result.coords("percentile", dim_coords=True): required_order.append(result.coord_dims("percentile")[0]) other_coords = [] for coord in result.dim_coords: if coord.name() not in ["realization", "percentile"]: other_coords.append(result.coord_dims(coord.name())[0]) required_order.extend(other_coords) result.transpose(required_order) return result
def test_basic(self): """Test that the method returns the expected cube type with coords""" result = self.plugin( CubeList([self.fg_cube, self.ltng_cube, self.precip_cube])) self.assertIsInstance(result, Cube) # We expect the threshold coordinate to have been removed. threshold_coord = find_threshold_coordinate(self.precip_cube).name() self.assertCountEqual( find_dimension_coordinate_mismatch(result, self.precip_cube), [threshold_coord], ) self.assertEqual(result.name(), "probability_of_rate_of_lightning_above_threshold") self.assertEqual(result.units, "1")
def test_no_mismatch(self): """Test if there is no mismatch between the dimension coordinates.""" cube = set_up_cube() result = find_dimension_coordinate_mismatch(cube, cube) self.assertIsInstance(result, list) self.assertFalse(result)
def process(self, cube: Cube, mask_cube: Optional[Cube] = None) -> Cube: """ Supply neighbourhood processing method, in order to smooth the input cube. Args: cube: Cube to apply a neighbourhood processing method to, in order to generate a smoother field. mask_cube: Cube containing the array to be used as a mask. Returns: Cube after applying a neighbourhood processing method, so that the resulting field is smoothed. """ if not getattr(self.neighbourhood_method, "run", None) or not callable( self.neighbourhood_method.run): msg = ("{} is not valid as a neighbourhood_method. " "Please choose a valid neighbourhood_method with a " "run method.".format(self.neighbourhood_method)) raise ValueError(msg) # Check if a dimensional realization coordinate exists. If so, the # cube is sliced, so that it becomes a scalar coordinate. try: cube.coord("realization", dim_coords=True) except iris.exceptions.CoordinateNotFoundError: slices_over_realization = [cube] else: slices_over_realization = cube.slices_over("realization") if np.isnan(cube.data).any(): raise ValueError("Error: NaN detected in input cube data") cubes_real = [] for cube_realization in slices_over_realization: if self.lead_times is None: cube_new = self.neighbourhood_method.run(cube_realization, self.radii, mask_cube=mask_cube) else: # Interpolate to find the radius at each required lead time. fp_coord = forecast_period_coord(cube_realization) fp_coord.convert_units("hours") required_radii = self._find_radii( cube_lead_times=fp_coord.points) cubes_time = iris.cube.CubeList([]) # Find the number of grid cells required for creating the # neighbourhood, and then apply the neighbourhood # processing method to smooth the field. for cube_slice, radius in zip( cube_realization.slices_over("time"), required_radii): cube_slice = self.neighbourhood_method.run( cube_slice, radius, mask_cube=mask_cube) cubes_time.append(cube_slice) cube_new = MergeCubes()(cubes_time) cubes_real.append(cube_new) if len(cubes_real) > 1: combined_cube = MergeCubes()(cubes_real, slice_over_realization=True) else: combined_cube = cubes_real[0] # Promote dimensional coordinates that used to be present. exception_coordinates = find_dimension_coordinate_mismatch( cube, combined_cube, two_way_mismatch=False) combined_cube = check_cube_coordinates( cube, combined_cube, exception_coordinates=exception_coordinates) return combined_cube
def run(self, cube: Cube, radius: float, mask_cube: Optional[Cube] = None) -> Cube: """ Method to apply a circular kernel to the data within the input cube in order to derive percentiles over the kernel. Args: cube: Cube containing array to apply processing to. radius: Radius in metres for use in specifying the number of grid cells used to create a circular neighbourhood. mask_cube: Cube containing the array to be used as a mask. Returns: Cube containing the percentile fields. Has percentile as an added dimension. """ if mask_cube is not None: msg = ("The use of a mask cube with a circular kernel is not " "yet implemented.") raise NotImplementedError(msg) # Check that the cube has an equal area grid. check_if_grid_is_equal_area(cube) # Take data array and identify X and Y axes indices grid_cell = distance_to_number_of_grid_cells(cube, radius) check_radius_against_distance(cube, radius) ranges_xy = np.array((grid_cell, grid_cell)) kernel = circular_kernel(ranges_xy, grid_cell, weighted_mode=False) # Loop over each 2D slice to reduce memory demand and derive # percentiles on the kernel. Will return an extra dimension. pctcubelist = iris.cube.CubeList() for slice_2d in cube.slices( ["projection_y_coordinate", "projection_x_coordinate"]): pctcubelist.append(self.pad_and_unpad_cube(slice_2d, kernel)) result = pctcubelist.merge_cube() exception_coordinates = find_dimension_coordinate_mismatch( cube, result, two_way_mismatch=False) result = check_cube_coordinates( cube, result, exception_coordinates=exception_coordinates) # Arrange cube, so that the coordinate order is: # realization, percentile, other coordinates. required_order = [] if result.coords("realization"): if result.coords("realization", dimensions=[]): result = iris.util.new_axis(result, "realization") required_order.append(result.coord_dims("realization")[0]) if result.coords("percentile"): required_order.append(result.coord_dims("percentile")[0]) other_coords = [] for coord in result.dim_coords: if coord.name() not in ["realization", "percentile"]: other_coords.append(result.coord_dims(coord.name())[0]) required_order.extend(other_coords) result.transpose(required_order) return result
def process(self, cube, mask_cube): """ 1. Iterate over the chosen coordinate within the mask_cube and apply the mask at each iteration to the cube that is to be neighbourhood processed. 2. Concatenate the cubes from each iteration together to create a single cube. Args: cube (iris.cube.Cube): Cube containing the array to which the square neighbourhood will be applied. mask_cube (iris.cube.Cube): Cube containing the array to be used as a mask. Returns: iris.cube.Cube: Cube containing the smoothed field after the square neighbourhood method has been applied when applying masking for each point along the coord_for_masking coordinate. The resulting cube is concatenated so that the dimension coordinates match the input cube. """ yname = cube.coord(axis="y").name() xname = cube.coord(axis="x").name() result_slices = iris.cube.CubeList([]) if self.collapse_weights is None: collapse_plugin = None else: collapse_plugin = CollapseMaskedNeighbourhoodCoordinate( self.coord_for_masking, self.collapse_weights) # Take 2D slices of the input cube for memory issues. prev_x_y_slice = None for x_y_slice in cube.slices([yname, xname]): if prev_x_y_slice is not None and np.array_equal( prev_x_y_slice.data, x_y_slice.data): # Use same result as last time! prev_result = result_slices[-1].copy() for coord in x_y_slice.coords(dim_coords=False): prev_result.coord(coord).points = coord.points.copy() result_slices.append(prev_result) continue prev_x_y_slice = x_y_slice cube_slices = iris.cube.CubeList([]) plugin = NeighbourhoodProcessing( self.neighbourhood_method, self.radii, lead_times=self.lead_times, weighted_mode=self.weighted_mode, sum_or_fraction=self.sum_or_fraction, re_mask=self.re_mask, ) # Apply each mask in in mask_cube to the 2D input slice. for cube_slice in mask_cube.slices_over(self.coord_for_masking): output_cube = plugin(x_y_slice, mask_cube=cube_slice) coord_object = cube_slice.coord(self.coord_for_masking).copy() output_cube.add_aux_coord(coord_object) output_cube = iris.util.new_axis(output_cube, self.coord_for_masking) cube_slices.append(output_cube) concatenated_cube = cube_slices.concatenate_cube() exception_coordinates = find_dimension_coordinate_mismatch( x_y_slice, concatenated_cube, two_way_mismatch=False) concatenated_cube = check_cube_coordinates( x_y_slice, concatenated_cube, exception_coordinates=exception_coordinates, ) if collapse_plugin: concatenated_cube = collapse_plugin(concatenated_cube) result_slices.append(concatenated_cube) result = result_slices.merge_cube() exception_coordinates = find_dimension_coordinate_mismatch( cube, result, two_way_mismatch=False) result = check_cube_coordinates( cube, result, exception_coordinates=exception_coordinates) return result
def process(self, cube: Cube, mask_cube: Cube) -> Cube: """ Apply neighbourhood processing with a mask to the input cube, collapsing the coord_for_masking if collapse_weights have been provided. Args: cube: Cube containing the array to which the square neighbourhood will be applied. mask_cube: Cube containing the array to be used as a mask. The data in this array is not an instance of numpy.ma.MaskedArray. Any sea points that should be ignored are set to zeros in every layer of the mask_cube. Returns: Cube containing the smoothed field after the square neighbourhood method has been applied when applying masking for each point along the coord_for_masking coordinate. The resulting cube is concatenated so that the dimension coordinates match the input cube. """ plugin = NeighbourhoodProcessing( self.neighbourhood_method, self.radii, lead_times=self.lead_times, weighted_mode=self.weighted_mode, sum_only=self.sum_only, re_mask=self.re_mask, ) yname = cube.coord(axis="y").name() xname = cube.coord(axis="x").name() result_slices = iris.cube.CubeList([]) # Take 2D slices of the input cube for memory issues. prev_x_y_slice = None for x_y_slice in cube.slices([yname, xname]): if prev_x_y_slice is not None and np.array_equal( prev_x_y_slice.data, x_y_slice.data): # Use same result as last time! prev_result = result_slices[-1].copy() for coord in x_y_slice.coords(dim_coords=False): prev_result.coord(coord).points = coord.points.copy() result_slices.append(prev_result) continue prev_x_y_slice = x_y_slice cube_slices = iris.cube.CubeList([]) # Apply each mask in in mask_cube to the 2D input slice. for mask_slice in mask_cube.slices_over(self.coord_for_masking): output_cube = plugin(x_y_slice, mask_cube=mask_slice) coord_object = mask_slice.coord(self.coord_for_masking).copy() output_cube.add_aux_coord(coord_object) output_cube = iris.util.new_axis(output_cube, self.coord_for_masking) cube_slices.append(output_cube) concatenated_cube = cube_slices.concatenate_cube() if self.collapse_weights is not None: concatenated_cube = self.collapse_mask_coord(concatenated_cube) result_slices.append(concatenated_cube) result = result_slices.merge_cube() # Promote any single value dimension coordinates if they were # dimension on the input cube. exception_coordinates = find_dimension_coordinate_mismatch( cube, result, two_way_mismatch=False) result = check_cube_coordinates( cube, result, exception_coordinates=exception_coordinates) return result
def process(self, cube): """ Supply neighbourhood processing method, in order to smooth the input cube. Parameters ---------- cube : Iris.cube.Cube Cube to apply a neighbourhood processing method to, in order to generate a smoother field. Returns ------- cube : Iris.cube.Cube Cube after applying a neighbourhood processing method, so that the resulting field is smoothed. """ if (not getattr(self.neighbourhood_method, "run", None) or not callable(self.neighbourhood_method.run)): msg = ("{} is not valid as a neighbourhood_method. " "Please choose a valid neighbourhood_method with a " "run method.".format(self.neighbourhood_method)) raise ValueError(msg) # Check if the realization coordinate exists. If there are multiple # values for the realization, then an exception is raised. Otherwise, # the cube is sliced, so that the realization becomes a scalar # coordinate. try: realiz_coord = cube.coord('realization') except iris.exceptions.CoordinateNotFoundError: if 'source_realizations' in cube.attributes: num_ens = len(cube.attributes['source_realizations']) else: num_ens = 1.0 slices_over_realization = [cube] else: num_ens = len(realiz_coord.points) slices_over_realization = cube.slices_over("realization") if 'source_realizations' in cube.attributes: msg = ("Realizations and attribute source_realizations " "should not both be set in input cube") raise ValueError(msg) if np.isnan(cube.data).any(): raise ValueError("Error: NaN detected in input cube data") cubelist = iris.cube.CubeList([]) for cube_realization in slices_over_realization: if self.lead_times is None: radius = self._find_radii(num_ens) cube_new = self.neighbourhood_method.run( cube_realization, radius) else: cube_lead_times = (find_required_lead_times(cube_realization)) # Interpolate to find the radius at each required lead time. required_radii = (self._find_radii( num_ens, cube_lead_times=cube_lead_times)) cubes = iris.cube.CubeList([]) # Find the number of grid cells required for creating the # neighbourhood, and then apply the neighbourhood # processing method to smooth the field. for cube_slice, radius in (zip( cube_realization.slices_over("time"), required_radii)): cube_slice = self.neighbourhood_method.run( cube_slice, radius) cube_slice = iris.util.new_axis(cube_slice, "time") cubes.append(cube_slice) cube_new = concatenate_cubes(cubes, coords_to_slice_over=["time"]) if cube_new.coords("realization", dim_coords=False): cube_new = iris.util.new_axis(cube_new, "realization") cubelist.append(cube_new) combined_cube = cubelist.concatenate_cube() # Promote dimensional coordinates that have been demoted to scalars. exception_coordinates = (find_dimension_coordinate_mismatch( cube, combined_cube, two_way_mismatch=False)) combined_cube = check_cube_coordinates( cube, combined_cube, exception_coordinates=exception_coordinates) return combined_cube