def test_error_range_greater_than_domain_size(self): """Test correct exception raised when the distance is larger than the corner-to-corner distance of the domain.""" distance = 42500.0 msg = "Distance of 42500.0m exceeds max domain distance of " with self.assertRaisesRegex(ValueError, msg): convert_distance_into_number_of_grid_cells(self.cube, distance)
def test_single_point_range_negative(self): """Test behaviour with a non-zero point with negative range.""" distance = -1.0 * self.DISTANCE msg = "distance of -6100.0m gives a negative cell extent" with self.assertRaisesRegexp(ValueError, msg): convert_distance_into_number_of_grid_cells( self.cube, distance, self.MAX_DISTANCE_IN_GRID_CELLS)
def test_single_point_lat_long(self): """Test behaviour for a single grid cell on lat long grid.""" cube = set_up_cube_lat_long() msg = "Invalid grid: projection_x/y coords required" with self.assertRaisesRegexp(ValueError, msg): convert_distance_into_number_of_grid_cells( cube, self.DISTANCE, self.MAX_DISTANCE_IN_GRID_CELLS)
def test_single_point_range_0(self): """Test behaviour with a non-zero point with zero range.""" distance = 5 msg = "Distance of 5m gives zero cell extent" with self.assertRaisesRegexp(ValueError, msg): convert_distance_into_number_of_grid_cells( self.cube, distance, self.MAX_DISTANCE_IN_GRID_CELLS)
def test_single_point_range_greater_than_domain(self): """Test correct exception raised when the distance is larger than the corner-to-corner distance of the domain.""" distance = 42500.0 msg = "Distance of 42500.0m exceeds max domain distance of " with self.assertRaisesRegexp(ValueError, msg): convert_distance_into_number_of_grid_cells( self.cube, distance, self.MAX_DISTANCE_IN_GRID_CELLS)
def test_single_point_range_lots(self): """Test behaviour with a non-zero point with unhandleable range.""" distance = 40000.0 max_distance_in_grid_cells = 10 msg = "distance of 40000.0m exceeds maximum grid cell extent" with self.assertRaisesRegexp(ValueError, msg): convert_distance_into_number_of_grid_cells( self.cube, distance, max_distance_in_grid_cells)
def test_error_outside_maximum_distance(self): """Test behaviour with a non-zero point with unhandleable range.""" distance = 40000.0 max_distance_in_grid_cells = 10 msg = "Distance of 40000.0m exceeds maximum permitted" with self.assertRaisesRegex(ValueError, msg): convert_distance_into_number_of_grid_cells( self.cube, distance, max_distance_in_grid_cells=max_distance_in_grid_cells)
def test_basic_distance_to_grid_cells_km_grid(self): """Test the distance-to-grid-cell conversion, grid in km.""" self.cube.coord("projection_x_coordinate").convert_units("kilometres") self.cube.coord("projection_y_coordinate").convert_units("kilometres") result = convert_distance_into_number_of_grid_cells( self.cube, self.DISTANCE, self.MAX_DISTANCE_IN_GRID_CELLS) self.assertEqual(result, (3, 3))
def test_basic_distance_to_grid_cells(self): """Test the distance in metres to grid cell conversion.""" result = convert_distance_into_number_of_grid_cells( self.cube, self.DISTANCE, max_distance_in_grid_cells=self.MAX_DISTANCE_IN_GRID_CELLS) self.assertEqual(result, (3, 3))
def run(self, cube, radius, mask_cube=None): """ Call the methods required to apply a square neighbourhood method to a cube. The steps undertaken are: 1. Set up cubes by determining, if the arrays are masked. 2. Pad the input array with a halo and then calculate the neighbourhood of the haloed array. 3. Remove the halo from the neighbourhooded array and deal with a mask, if required. Args: cube (iris.cube.Cube): Cube containing the array to which the square neighbourhood will be applied. radius (float): Radius in metres for use in specifying the number of grid cells used to create a square neighbourhood. mask_cube (iris.cube.Cube): Cube containing the array to be used as a mask. Returns: neighbourhood_averaged_cube (iris.cube.Cube): Cube containing the smoothed field after the square neighbourhood method has been applied. """ # If the data is masked, the mask will be processed as well as the # original_data * mask array. original_attributes = cube.attributes original_methods = cube.cell_methods grid_cells_x = (convert_distance_into_number_of_grid_cells( cube, radius, max_distance_in_grid_cells=MAX_RADIUS_IN_GRID_CELLS)) grid_cells_y = grid_cells_x result_slices = iris.cube.CubeList() for cube_slice in cube.slices( [cube.coord(axis='y'), cube.coord(axis='x')]): (cube_slice, mask, nan_array) = (self.set_up_cubes_to_be_neighbourhooded( cube_slice, mask_cube)) neighbourhood_averaged_cube = ( self._pad_and_calculate_neighbourhood(cube_slice, mask, grid_cells_x, grid_cells_y)) neighbourhood_averaged_cube = (self._remove_padding_and_mask( neighbourhood_averaged_cube, cube_slice, mask, grid_cells_x, grid_cells_y)) neighbourhood_averaged_cube.data[nan_array.astype(bool)] = np.nan result_slices.append(neighbourhood_averaged_cube) neighbourhood_averaged_cube = result_slices.merge_cube() neighbourhood_averaged_cube.cell_methods = original_methods neighbourhood_averaged_cube.attributes = original_attributes neighbourhood_averaged_cube = check_cube_coordinates( cube, neighbourhood_averaged_cube) return neighbourhood_averaged_cube
def test_max_distance(self): """ Test the distance in metres to grid cell conversion within a maximum distance in grid cells. """ result = convert_distance_into_number_of_grid_cells( self.cube, self.DISTANCE, max_distance_in_grid_cells=50) self.assertEqual(result, 3)
def remove_cube_halo(cube, halo_radius): """ Remove halo of halo_radius from a cube. This function converts the halo radius into the number of grid points in the x and y coordinate that need to be removed. It then calls remove_halo_from_cube which only acts on a cube with x and y coordinates so we need to slice the cube and them merge the cube back together ensuring the resulting cube has the same dimension coordinates. Args: cube (iris.cube.Cube): Cube on extended grid halo_radius (float): Size of border to remove, in metres Returns: iris.cube.Cube: New cube with the halo removed. """ halo_size_x = convert_distance_into_number_of_grid_cells(cube, halo_radius, axis='x') halo_size_y = convert_distance_into_number_of_grid_cells(cube, halo_radius, axis='y') result_slices = iris.cube.CubeList() for cube_slice in cube.slices([cube.coord(axis='y'), cube.coord(axis='x')]): cube_halo = remove_halo_from_cube(cube_slice, halo_size_x, halo_size_y) result_slices.append(cube_halo) result = result_slices.merge_cube() # re-promote any scalar dimensions lost in slice / merge req_dims = [coord.name() for coord in cube.coords(dim_coords=True)] present_dims = [coord.name() for coord in result.coords(dim_coords=True)] for coord in req_dims: if coord not in present_dims: result = iris.util.new_axis(result, coord) # re-order (needed if scalar dimensions have been re-added) enforce_coordinate_ordering(result, req_dims) return result
def test_distance_to_grid_cells_other_axis(self): """Test the distance in metres to grid cell conversion along the y-axis.""" self.cube.coord( axis='y').points = 0.5 * self.cube.coord(axis='y').points result = convert_distance_into_number_of_grid_cells(self.cube, self.DISTANCE, axis='y') self.assertEqual(result, 6)
def test_basic_distance_to_grid_cells_different_max_distance(self): """ Test the distance in metres to grid cell conversion for an alternative max distance in grid cells. """ max_distance_in_grid_cells = 50 result = convert_distance_into_number_of_grid_cells( self.cube, self.DISTANCE, max_distance_in_grid_cells) self.assertEqual(result, (3, 3))
def run(self, cube, radius, mask_cube=None): """ Call the methods required to apply a square neighbourhood method to a cube. The steps undertaken are: 1. Set up cubes by determining, if the arrays are masked. 2. Pad the input array with a halo and then calculate the neighbourhood of the haloed array. 3. Remove the halo from the neighbourhooded array and deal with a mask, if required. Args: cube (Iris.cube.Cube): Cube containing the array to which the square neighbourhood will be applied. radius (Float): Radius in metres for use in specifying the number of grid cells used to create a square neighbourhood. Keyword Args: mask_cube (Iris.cube.Cube): Cube containing the array to be used as a mask. Returns: neighbourhood_averaged_cube (Iris.cube.Cube): Cube containing the smoothed field after the square neighbourhood method has been applied. """ # If the data is masked, the mask will be processed as well as the # original_data * mask array. original_attributes = cube.attributes original_methods = cube.cell_methods grid_cells_x, grid_cells_y = ( convert_distance_into_number_of_grid_cells( cube, radius, MAX_RADIUS_IN_GRID_CELLS)) cubes_to_sum = (self._set_up_cubes_to_be_neighbourhooded( cube, mask_cube)) neighbourhood_averaged_cubes = (self._pad_and_calculate_neighbourhood( cubes_to_sum, grid_cells_x, grid_cells_y)) neighbourhood_averaged_cube = (self._remove_padding_and_mask( neighbourhood_averaged_cubes, cubes_to_sum, cube.name(), grid_cells_x, grid_cells_y)) neighbourhood_averaged_cube.cell_methods = original_methods neighbourhood_averaged_cube.attributes = original_attributes neighbourhood_averaged_cube = check_cube_coordinates( cube, neighbourhood_averaged_cube) return neighbourhood_averaged_cube
def remove_cube_halo(cube, halo_radius): """ Remove halo of halo_radius from a cube. This function converts the halo radius into the number of grid points in the x and y coordinate that need to be removed. It then calls remove_halo_from_cube which only acts on a cube with x and y coordinates so we need to slice the cube and them merge the cube back together ensuring the resulting cube has the same dimension coordinates. Args: cube (iris.cube.Cube): Cube on extended grid halo_radius (float): Size of border to remove, in metres Returns: result (iris.cube.Cube): New cube with the halo removed. """ halo_size_x, halo_size_y = convert_distance_into_number_of_grid_cells( cube, halo_radius) result_slices = iris.cube.CubeList() for cube_slice in cube.slices([cube.coord(axis='y'), cube.coord(axis='x')]): cube_halo = remove_halo_from_cube(cube_slice, halo_size_x, halo_size_y) result_slices.append(cube_halo) result = result_slices.merge_cube() req_coords = [] for coord in cube.coords(dim_coords=True): req_coords.append(coord.name()) result = enforce_coordinate_ordering( result, req_coords, promote_scalar=True) return result
def _update_spatial_weights(self, cube, weights, fuzzy_length): """ Update weights using spatial information Args: cube (iris.cube.Cube): Cube of input data to be blended weights (iris.cube.Cube): Initial 1D cube of weights scaled by self.weighting_coord fuzzy_length (float): Distance (in metres) over which to smooth weights at domain boundaries Returns: weights (iris.cube.Cube): Updated 3D cube of spatially-varying weights """ check_if_grid_is_equal_area(cube) grid_cells_x, _ = convert_distance_into_number_of_grid_cells( cube, fuzzy_length, int_grid_cells=False) SpatialWeightsPlugin = SpatiallyVaryingWeightsFromMask(grid_cells_x) weights = SpatialWeightsPlugin.process(cube, weights, self.blend_coord) return weights
def test_basic_no_limit(self): """Test the distance in metres to grid cell conversion still works when the maximum distance limit is not explicitly set.""" result = convert_distance_into_number_of_grid_cells( self.cube, self.DISTANCE) self.assertEqual(result, (3, 3))
def main(argv=None): """Load in arguments and ensure they are set correctly. Then load in the data to blend and calculate default weights using the method chosen before carrying out the blending.""" parser = ArgParser( description='Calculate the default weights to apply in weighted ' 'blending plugins using the ChooseDefaultWeightsLinear or ' 'ChooseDefaultWeightsNonLinear plugins. Then apply these ' 'weights to the dataset using the BasicWeightedAverage plugin.' ' Required for ChooseDefaultWeightsLinear: y0val and ynval.' ' Required for ChooseDefaultWeightsNonLinear: cval.' ' Required for ChooseWeightsLinear with dict: wts_dict.') parser.add_argument('--wts_calc_method', metavar='WEIGHTS_CALCULATION_METHOD', choices=['linear', 'nonlinear', 'dict'], default='linear', help='Method to use to calculate ' 'weights used in blending. "linear" (default): ' 'calculate linearly varying blending weights. ' '"nonlinear": calculate blending weights that decrease' ' exponentially with increasing blending coordinate. ' '"dict": calculate weights using a dictionary passed ' 'in as a command line argument.') parser.add_argument('coordinate', type=str, metavar='COORDINATE_TO_AVERAGE_OVER', help='The coordinate over which the blending ' 'will be applied.') parser.add_argument('--coordinate_unit', metavar='UNIT_STRING', default='hours since 1970-01-01 00:00:00', help='Units for blending coordinate. Default= ' 'hours since 1970-01-01 00:00:00') parser.add_argument('--calendar', metavar='CALENDAR', help='Calendar for time coordinate. Default=gregorian') parser.add_argument('--cycletime', metavar='CYCLETIME', type=str, help='The forecast reference time to be used after ' 'blending has been applied, in the format ' 'YYYYMMDDTHHMMZ. If not provided, the blended file ' 'will take the latest available forecast reference ' 'time from the input cubes supplied.') parser.add_argument('--model_id_attr', metavar='MODEL_ID_ATTR', type=str, default="mosg__model_configuration", help='The name of the netCDF file attribute to be ' 'used to identify the source model for ' 'multi-model blends. Default assumes Met Office ' 'model metadata. Must be present on all input ' 'files if blending over models.') parser.add_argument('--spatial_weights_from_mask', action='store_true', default=False, help='If set this option will result in the generation' ' of spatially varying weights based on the' ' masks of the data we are blending. The' ' one dimensional weights are first calculated ' ' using the chosen weights calculation method,' ' but the weights will then be adjusted spatially' ' based on where there is masked data in the data' ' we are blending. The spatial weights are' ' calculated using the' ' SpatiallyVaryingWeightsFromMask plugin.') parser.add_argument('weighting_mode', metavar='WEIGHTED_BLEND_MODE', choices=['weighted_mean', 'weighted_maximum'], help='The method used in the weighted blend. ' '"weighted_mean": calculate a normal weighted' ' mean across the coordinate. ' '"weighted_maximum": multiplies the values in the' ' coordinate by the weights, and then takes the' ' maximum.') parser.add_argument('input_filepaths', metavar='INPUT_FILES', nargs="+", help='Paths to input files to be blended.') parser.add_argument('output_filepath', metavar='OUTPUT_FILE', help='The output path for the processed NetCDF.') spatial = parser.add_argument_group( 'Spatial weights from mask options', 'Options for calculating the spatial weights using the ' 'SpatiallyVaryingWeightsFromMask plugin.') spatial.add_argument('--fuzzy_length', metavar='FUZZY_LENGTH', type=float, default=20000, help='When calculating spatially varying weights we' ' can smooth the weights so that areas close to' ' areas that are masked have lower weights than' ' those further away. This fuzzy length controls' ' the scale over which the weights are smoothed.' ' The fuzzy length is in terms of m, the' ' default is 20km. This distance is then' ' converted into a number of grid squares,' ' which does not have to be an integer. Assumes' ' the grid spacing is the same in the x and y' ' directions, and raises an error if this is not' ' true. See SpatiallyVaryingWeightsFromMask for' ' more detail.') linear = parser.add_argument_group( 'linear weights options', 'Options for the linear weights ' 'calculation in ' 'ChooseDefaultWeightsLinear') linear.add_argument('--y0val', metavar='LINEAR_STARTING_POINT', type=float, help='The relative value of the weighting start point ' '(lowest value of blend coord) for choosing default ' 'linear weights. This must be a positive float or 0.') linear.add_argument('--ynval', metavar='LINEAR_END_POINT', type=float, help='The relative value of the weighting ' 'end point (highest value of blend coord) for choosing' ' default linear weights. This must be a positive ' 'float or 0. Note that if blending over forecast ' 'reference time, ynval >= y0val would normally be ' 'expected (to give greater weight to the more recent ' 'forecast).') nonlinear = parser.add_argument_group( 'nonlinear weights options', 'Options for the non-linear ' 'weights calculation in ' 'ChooseDefaultWeightsNonLinear') nonlinear.add_argument('--cval', metavar='NON_LINEAR_FACTOR', type=float, help='Factor used to determine how skewed the ' 'non linear weights will be. ' 'A value of 1 implies equal weighting. If not ' 'set, a default value of cval=0.85 is set.') wts_dict = parser.add_argument_group( 'dict weights options', 'Options for linear weights to be ' 'calculated based on parameters ' 'read from a json file dict') wts_dict.add_argument('--wts_dict', metavar='WEIGHTS_DICTIONARY', help='Path to json file containing dictionary from ' 'which to calculate blending weights. Dictionary ' 'format is as specified in the improver.blending.' 'weights.ChooseWeightsLinear plugin.') wts_dict.add_argument('--weighting_coord', metavar='WEIGHTING_COORD', default='forecast_period', help='Name of ' 'coordinate over which linear weights should be ' 'scaled. This coordinate must be avilable in the ' 'weights dictionary.') args = parser.parse_args(args=argv) # if the linear weights method is called with non-linear args or vice # versa, exit with error if (args.wts_calc_method == "linear") and args.cval: parser.wrong_args_error('cval', 'linear') if ((args.wts_calc_method == "nonlinear") and np.any([args.y0val, args.ynval])): parser.wrong_args_error('y0val, ynval', 'non-linear') if (args.wts_calc_method == "dict") and not args.wts_dict: parser.error('Dictionary is required if --wts_calc_method="dict"') # set blending coordinate units if "time" in args.coordinate: coord_unit = Unit(args.coordinate_unit, args.calendar) elif args.coordinate_unit != 'hours since 1970-01-01 00:00:00.': coord_unit = args.coordinate_unit else: coord_unit = 'no_unit' # For blending across models, only blending across "model_id" is directly # supported. This is because the blending coordinate must be sortable, in # order to ensure that the data cube and the weights cube have coordinates # in the same order for blending. Whilst the model_configuration is # sortable itself, as it is associated with model_id, which is the # dimension coordinate, sorting the model_configuration coordinate can # result in the model_id coordinate becoming non-monotonic. As dimension # coordinates must be monotonic, this leads to the model_id coordinate # being demoted to an auxiliary coordinate. Therefore, for simplicity # model_id is used as the blending coordinate, instead of # model_configuration. # TODO: Support model_configuration as a blending coordinate directly. if args.coordinate == "model_configuration": blend_coord = "model_id" dict_coord = "model_configuration" else: blend_coord = args.coordinate dict_coord = args.coordinate # load cubes to be blended cubelist = load_cubelist(args.input_filepaths) # determine whether or not to equalise forecast periods for model # blending weights calculation weighting_coord = (args.weighting_coord if args.weighting_coord else "forecast_period") # prepare cubes for weighted blending merger = MergeCubesForWeightedBlending(blend_coord, weighting_coord=weighting_coord, model_id_attr=args.model_id_attr) cube = merger.process(cubelist, cycletime=args.cycletime) # if the coord for blending does not exist or has only one value, # update metadata only coord_names = [coord.name() for coord in cube.coords()] if (blend_coord not in coord_names) or (len( cube.coord(blend_coord).points) == 1): result = cube.copy() conform_metadata(result, cube, blend_coord, cycletime=args.cycletime) # raise a warning if this happened because the blend coordinate # doesn't exist if blend_coord not in coord_names: warnings.warn('Blend coordinate {} is not present on input ' 'data'.format(blend_coord)) # otherwise, calculate weights and blend across specified dimension else: weights = calculate_blending_weights( cube, blend_coord, args.wts_calc_method, wts_dict=args.wts_dict, weighting_coord=args.weighting_coord, coord_unit=coord_unit, y0val=args.y0val, ynval=args.ynval, cval=args.cval, dict_coord=dict_coord) if args.spatial_weights_from_mask: check_if_grid_is_equal_area(cube) grid_cells_x, _ = convert_distance_into_number_of_grid_cells( cube, args.fuzzy_length, int_grid_cells=False) SpatialWeightsPlugin = SpatiallyVaryingWeightsFromMask( grid_cells_x) weights = SpatialWeightsPlugin.process(cube, weights, blend_coord) # blend across specified dimension BlendingPlugin = WeightedBlendAcrossWholeDimension( blend_coord, args.weighting_mode, cycletime=args.cycletime) result = BlendingPlugin.process(cube, weights=weights) save_netcdf(result, args.output_filepath)
def test_basic_distance_to_grid_cells(self): """Test the distance in metres to grid cell conversion along the x-axis (default).""" result = convert_distance_into_number_of_grid_cells( self.cube, self.DISTANCE) self.assertEqual(result, 3)
def test_basic_distance_to_grid_cells_float(self): """Test the distance in metres to grid cell conversion.""" result = convert_distance_into_number_of_grid_cells( self.cube, self.DISTANCE, int_grid_cells=False) self.assertEqual(result, 3.05)
def test_error_zero_grid_cell_range(self): """Test behaviour with a non-zero point with zero range.""" distance = 5 msg = "Distance of 5m gives zero cell extent" with self.assertRaisesRegex(ValueError, msg): convert_distance_into_number_of_grid_cells(self.cube, distance)
def test_error_negative_distance(self): """Test behaviour with a non-zero point with negative range.""" distance = -1.0 * self.DISTANCE msg = "Please specify a positive distance in metres" with self.assertRaisesRegex(ValueError, msg): convert_distance_into_number_of_grid_cells(self.cube, distance)