def test_not_in_vicinity(self): """Test for no change if the matching point is too far away.""" # We need larger arrays for this. # Define 5 x 5 arrays with output sea point at [1, 1] and input sea # point at [4, 4]. The alternative value of 0.5 at [4, 4] should not # be selected with a small vicinity_radius. self.plugin = RegridLandSea(vicinity_radius=2200.) cube = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 1, 1), ))) self.plugin.output_land = cube.copy() self.plugin.nearest_cube = cube.copy() self.plugin.nearest_cube.data[4, 4] = 0.5 self.plugin.output_cube = self.plugin.nearest_cube.copy() self.plugin.input_land = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 4, 4), ))) output_cube = self.plugin.output_cube.copy() self.plugin.correct_where_input_true(0) self.assertArrayEqual(output_cube.data, self.plugin.output_cube.data)
def setUp(self): """Create a class-object containing the necessary cubes. All cubes are on the target grid. Here this is defined as a 3x3 grid. The grid contains ones everywhere except the centre point (a zero). The output_cube has a value of 0.5 at [0, 1]. The move_sea_point cube has the zero value at [0, 1] instead of [1, 1], this allows it to be used in place of input_land to trigger the expected behaviour in the function. """ self.plugin = RegridLandSea(vicinity_radius=2200.) cube = squeeze( set_up_cube(num_grid_points=3, zero_point_indices=((0, 0, 1, 1), ))) self.plugin.input_land = cube.copy() self.plugin.output_land = cube.copy() self.plugin.nearest_cube = cube.copy() self.plugin.nearest_cube.data[0, 1] = 0.5 self.plugin.output_cube = self.plugin.nearest_cube.copy() self.move_sea_point = squeeze( set_up_cube(num_grid_points=3, zero_point_indices=((0, 0, 0, 1), )))
def setUp(self): """Create a class-object containing the necessary cubes. All cubes are on the target grid. Here this is defined as a 5x5 grid. The cubes have values of one everywhere except: input_land: zeroes (sea points) at [0, 1], [4, 4] output_land: zeroes (sea points) at [0, 0], [1, 1] input_cube: 0. at [1, 1]; 0.5 at [0, 1]; 0.1 at [4, 4] These should trigger all the behavior we expect. """ self.plugin = RegridLandSea(vicinity_radius=2200.) self.output_land = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 1, 1), (0, 0, 0, 0)))) self.cube = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 1, 1), ))) self.cube.data[0, 1] = 0.5 self.cube.data[4, 4] = 0.1 self.input_land = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 0, 1), (0, 0, 4, 4)))) # Lat-lon coords for reprojection # These coords result in a 1:1 regridding with the above cubes. x_coord = DimCoord(np.linspace(-3.281, -3.153, 5), standard_name='longitude', units='degrees', coord_system=ELLIPSOID) y_coord = DimCoord(np.linspace(54.896, 54.971, 5), standard_name='latitude', units='degrees', coord_system=ELLIPSOID) self.input_land_ll = Cube(self.input_land.data, long_name='land_sea_mask', units='1', dim_coords_and_dims=[(y_coord, 0), (x_coord, 1)])
def test_basic(self): """Test that instantiating the class results in an object with expected variables.""" expected_members = { 'nearest_cube': None, 'input_land': None, 'output_land': None, 'output_cube': None } result = RegridLandSea() members = { attr: getattr(result, attr) for attr in dir(result) if not callable(getattr(result, attr)) and not attr.startswith("__") } regridder = members.pop('regridder') vicinity = members.pop('vicinity') self.assertDictEqual(members, expected_members) self.assertTrue(isinstance(regridder, iris.analysis.Nearest)) self.assertTrue(isinstance(vicinity, OccurrenceWithinVicinity))
def process(output_data, target_grid=None, source_landsea=None, metadata_dict=None, regrid_mode='bilinear', extrapolation_mode='nanmask', landmask_vicinity=25000, fix_float64=False): """Standardises a cube by one or more of regridding, updating meta-data etc Standardise a source cube. Available options are regridding (bi-linear or nearest-neighbour, optionally with land-mask awareness), updating meta-data and converting float64 data to float32. A check for float64 data compliance can be made by only specifying a source cube with no other arguments. Args: output_data (iris.cube.Cube): Output cube. If the only argument, then it is checked bor float64 data. target_grid (iris.cube.Cube): If specified, then regridding of the source against the target grid is enabled. If also using landmask-aware regridding then this must be land_binary_mask data. Default is None. source_landsea (iris.cube.Cube): A cube describing the land_binary_mask on the source-grid if coastline-aware regridding is required. Default is None. metadata_dict (dict): Dictionary containing required changes that will be applied to the metadata. Default is None. regrid_mode (str): Selects which regridding techniques to use. Default uses iris.analysis.Linear(); "nearest" uses Nearest() (Use for less continuous fields, e.g precipitation.); "nearest-with-mask" ensures that target data are sources from points with the same mask value (Use for coast-line-dependant variables like temperature). extrapolation_mode (str): Mode to use for extrapolating data into regions beyond the limits of the source_data domain. Refer to online documentation for iris.analysis. Modes are - extrapolate -The extrapolation points will take their values from the nearest source point. nan - The extrapolation points will be set to NaN. error - A ValueError exception will be raised notifying an attempt to extrapolate. mask - The extrapolation points will always be masked, even if the source data is not a MaskedArray. nanmask - If the source data is a MaskedArray the extrapolation points will be masked. Otherwise they will be set to NaN. Defaults is 'nanmask'. landmask_vicinity (float): Radius of vicinity to search for a coastline, in metres. Defaults is 25000 m fix_float64 (bool): If True, checks and fixes cube for float64 data. Without this option an exception will be raised if float64 data is found but no fix applied. Default is False. Returns: output_data (iris.cube.Cube): Processed cube. Raises: ValueError: If source landsea is supplied but regrid mode not nearest-with-mask. ValueError: If source landsea is supplied but not target grid. ValueError: If regrid_mode is "nearest-with-mask" but no landmask cube has been provided. Warns: warning: If the 'source_landsea' did not have a cube named land_binary_mask. warning: If the 'target_grid' did not have a cube named land_binary_mask. """ if (source_landsea and "nearest-with-mask" not in regrid_mode): msg = ("Land-mask file supplied without appropriate regrid_mode. " "Use --regrid_mode=nearest-with-mask.") raise ValueError(msg) if source_landsea and not target_grid: msg = ("Cannot specify input_landmask_filepath without " "target_grid_filepath") raise ValueError(msg) # Process # Re-grid with options: check_cube_not_float64(output_data, fix=fix_float64) # if a target grid file has been specified, then regrid optionally # applying float64 data check, metadata change, Iris nearest and # extrapolation mode as required. if target_grid: regridder = iris.analysis.Linear( extrapolation_mode=extrapolation_mode) if regrid_mode in ["nearest", "nearest-with-mask"]: regridder = iris.analysis.Nearest( extrapolation_mode=extrapolation_mode) output_data = output_data.regrid(target_grid, regridder) if regrid_mode in ["nearest-with-mask"]: if not source_landsea: msg = ("An argument has been specified that requires an input " "landmask cube but none has been provided") raise ValueError(msg) if "land_binary_mask" not in source_landsea.name(): msg = ("Expected land_binary_mask in input_landmask cube " "but found {}".format(repr(source_landsea))) warnings.warn(msg) if "land_binary_mask" not in target_grid.name(): msg = ("Expected land_binary_mask in target_grid cube " "but found {}".format(repr(target_grid))) warnings.warn(msg) output_data = RegridLandSea( vicinity_radius=landmask_vicinity).process( output_data, source_landsea, target_grid) target_grid_attributes = ( {k: v for (k, v) in target_grid.attributes.items() if 'mosg__' in k or 'institution' in k}) amend_metadata(output_data, attributes=target_grid_attributes) # Change metadata only option: # if output file path and json metadata file specified, # change the metadata if metadata_dict: output_data = amend_metadata(output_data, **metadata_dict) check_cube_not_float64(output_data, fix=fix_float64) return output_data
def main(argv=None): """ Standardise a source cube. Available options are regridding (bilinear or nearest-neighbour, optionally with land-mask awareness), updating meta-data and converting float64 data to float32. A check for float64 data compliance can be made by only specify a source NetCDF file with no other arguments. """ parser = ArgParser( description='Standardise a source data cube. Three main options are ' 'available; fixing float64 data, regridding and updating ' 'metadata. If regridding then additional options are ' 'available to use bilinear or nearest-neighbour ' '(optionally with land-mask awareness) modes. If only a ' 'source file is specified with no other arguments, then ' 'an exception will be raised if float64 data are found on ' 'the source.') parser.add_argument('source_data_filepath', metavar='SOURCE_DATA', help='A cube of data that is to be standardised and ' 'optionally fixed for float64 data, regridded ' 'and meta data changed') parser.add_argument("--output_filepath", metavar="OUTPUT_FILE", default=None, help="The output path for the processed NetCDF. " "If only a source file is specified and no " "output file, then the source will be checked" "for float64 data.") regrid_group = parser.add_argument_group("Regridding options") regrid_group.add_argument( "--target_grid_filepath", metavar="TARGET_GRID", help=('If specified then regridding of the source ' 'against the target grid is enabled. If also using ' 'landmask-aware regridding, then this must be land_binary_mask ' 'data.')) regrid_group.add_argument( "--regrid_mode", default='bilinear', choices=['bilinear', 'nearest', 'nearest-with-mask'], help=('Selects which regridding technique to use. Default uses ' 'iris.analysis.Linear(); "nearest" uses Nearest() (Use for less ' 'continuous fields, e.g. precipitation.); "nearest-with-mask" ' 'ensures that target data are sourced from points with the same ' 'mask value (Use for coast-line-dependent variables like ' 'temperature).')) regrid_group.add_argument( "--extrapolation_mode", default='nanmask', help='Mode to use for extrapolating data into regions ' 'beyond the limits of the source_data domain. ' 'Refer to online documentation for iris.analysis. ' 'Modes are: ' 'extrapolate - The extrapolation points will ' 'take their value from the nearest source point. ' 'nan - The extrapolation points will be be ' 'set to NaN. ' 'error - A ValueError exception will be raised, ' 'notifying an attempt to extrapolate. ' 'mask - The extrapolation points will always be ' 'masked, even if the source data is not a ' 'MaskedArray. ' 'nanmask - If the source data is a MaskedArray ' 'the extrapolation points will be masked. ' 'Otherwise they will be set to NaN. ' 'Defaults to nanmask.') regrid_group.add_argument( "--input_landmask_filepath", metavar="INPUT_LANDMASK_FILE", help=("A path to a NetCDF file describing the land_binary_mask on " "the source-grid if coastline-aware regridding is required.")) regrid_group.add_argument( "--landmask_vicinity", metavar="LANDMASK_VICINITY", default=25000., type=float, help=("Radius of vicinity to search for a coastline, in metres. " "Default value; 25000 m")) parser.add_argument("--fix_float64", action='store_true', default=False, help="Check and fix cube for float64 data. Without " "this option an exception will be raised if " "float64 data is found but no fix applied.") parser.add_argument("--json_file", metavar="JSON_FILE", default=None, help='Filename for the json file containing required ' 'changes that will be applied ' 'to the metadata. Defaults to None.') args = parser.parse_args(args=argv) if args.target_grid_filepath or args.json_file or args.fix_float64: if not args.output_filepath: msg = ("An argument has been specified that requires an output " "filepath but none has been provided") raise ValueError(msg) if (args.input_landmask_filepath and "nearest-with-mask" not in args.regrid_mode): msg = ("Land-mask file supplied without appropriate regrid_mode. " "Use --regrid_mode=nearest-with-mask.") raise ValueError(msg) if args.input_landmask_filepath and not args.target_grid_filepath: msg = ("Cannot specify input_landmask_filepath without " "target_grid_filepath") raise ValueError(msg) # source file data path is a mandatory argument output_data = load_cube(args.source_data_filepath) if args.fix_float64: check_cube_not_float64(output_data, fix=True) else: check_cube_not_float64(output_data, fix=False) # Re-grid with options: # if a target grid file has been specified, then regrid optionally # applying float64 data check, metadata change, Iris nearest and # extrapolation mode as required. if args.target_grid_filepath: target_grid = load_cube(args.target_grid_filepath) regridder = iris.analysis.Linear( extrapolation_mode=args.extrapolation_mode) if args.regrid_mode in ["nearest", "nearest-with-mask"]: regridder = iris.analysis.Nearest( extrapolation_mode=args.extrapolation_mode) output_data = output_data.regrid(target_grid, regridder) if args.regrid_mode in ["nearest-with-mask"]: if not args.input_landmask_filepath: msg = ("An argument has been specified that requires an input " "landmask filepath but none has been provided") raise ValueError(msg) source_landsea = load_cube(args.input_landmask_filepath) if "land_binary_mask" not in source_landsea.name(): msg = ("Expected land_binary_mask in input_landmask_filepath " "but found {}".format(repr(source_landsea))) warnings.warn(msg) if "land_binary_mask" not in target_grid.name(): msg = ("Expected land_binary_mask in target_grid_filepath " "but found {}".format(repr(target_grid))) warnings.warn(msg) output_data = RegridLandSea( vicinity_radius=args.landmask_vicinity).process( output_data, source_landsea, target_grid) target_grid_attributes = ({ k: v for (k, v) in target_grid.attributes.items() if 'mosg__' in k or 'institution' in k }) amend_metadata(output_data, attributes=target_grid_attributes) # Change metadata only option: # if output file path and json metadata file specified, # change the metadata if args.json_file: with open(args.json_file, 'r') as input_file: metadata_dict = json.load(input_file) output_data = amend_metadata(output_data, **metadata_dict) # Check and fix for float64 data only option: if args.fix_float64: check_cube_not_float64(output_data, fix=True) if args.output_filepath: save_netcdf(output_data, args.output_filepath)
def test_vicinity_arg(self): """Test with vicinity_radius argument.""" result = RegridLandSea(vicinity_radius=30000.) vicinity = getattr(result, 'vicinity') self.assertTrue(isinstance(vicinity, OccurrenceWithinVicinity)) self.assertEqual(vicinity.distance, 30000.)
def test_extrap_arg_error(self): """Test with invalid extrapolation_mode argument.""" msg = "Extrapolation mode 'not_valid' not supported" with self.assertRaisesRegex(ValueError, msg): RegridLandSea(extrapolation_mode="not_valid")
def test_extrap_arg(self): """Test with extrapolation_mode argument.""" result = RegridLandSea(extrapolation_mode="mask") regridder = getattr(result, 'regridder') self.assertTrue(isinstance(regridder, iris.analysis.Nearest))
class Test_process(IrisTest): """Tests the process method of the RegridLandSea class.""" def setUp(self): """Create a class-object containing the necessary cubes. All cubes are on the target grid. Here this is defined as a 5x5 grid. The cubes have values of one everywhere except: input_land: zeroes (sea points) at [0, 1], [4, 4] output_land: zeroes (sea points) at [0, 0], [1, 1] input_cube: 0. at [1, 1]; 0.5 at [0, 1]; 0.1 at [4, 4] These should trigger all the behavior we expect. """ self.plugin = RegridLandSea(vicinity_radius=2200.) self.output_land = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 1, 1), (0, 0, 0, 0)))) self.cube = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 1, 1), ))) self.cube.data[0, 1] = 0.5 self.cube.data[4, 4] = 0.1 self.input_land = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 0, 1), (0, 0, 4, 4)))) # Lat-lon coords for reprojection # These coords result in a 1:1 regridding with the above cubes. x_coord = DimCoord(np.linspace(-3.281, -3.153, 5), standard_name='longitude', units='degrees', coord_system=ELLIPSOID) y_coord = DimCoord(np.linspace(54.896, 54.971, 5), standard_name='latitude', units='degrees', coord_system=ELLIPSOID) self.input_land_ll = Cube(self.input_land.data, long_name='land_sea_mask', units='1', dim_coords_and_dims=[(y_coord, 0), (x_coord, 1)]) # The warning messages are internal to the iris.analysis module v2.2.0. @ManageWarnings(ignored_messages=["Using a non-tuple sequence for "], warning_types=[FutureWarning]) def test_basic(self): """Test that the expected changes occur and meta-data are unchanged.""" expected = self.cube.data.copy() # Output sea-point populated with data from input sea-point: expected[0, 0] = 0.5 # Output sea-point populated with data from input sea-point: expected[1, 1] = 0.5 # Output land-point populated with data from input land-point: expected[0, 1] = 1. # Output land-point populated with data from input sea-point due to # vicinity-constraint: expected[4, 4] = 1. result = self.plugin.process(self.cube, self.input_land, self.output_land) self.assertIsInstance(result, Cube) self.assertArrayEqual(result.data, expected) self.assertDictEqual(result.attributes, self.cube.attributes) self.assertEqual(result.name(), self.cube.name()) @ManageWarnings(ignored_messages=["Using a non-tuple sequence for "], warning_types=[FutureWarning]) def test_with_regridding(self): """Test when input grid is on a different projection.""" self.input_land = self.input_land_ll expected = self.cube.data.copy() # Output sea-point populated with data from input sea-point: expected[0, 0] = 0.5 # Output sea-point populated with data from input sea-point: expected[1, 1] = 0.5 # Output land-point populated with data from input land-point: expected[0, 1] = 1. # Output land-point populated with data from input sea-point due to # vicinity-constraint: expected[4, 4] = 1. result = self.plugin.process(self.cube, self.input_land, self.output_land) self.assertIsInstance(result, Cube) self.assertArrayEqual(result.data, expected) self.assertDictEqual(result.attributes, self.cube.attributes) self.assertEqual(result.name(), self.cube.name()) @ManageWarnings(ignored_messages=["Using a non-tuple sequence for "], warning_types=[FutureWarning]) def test_multi_realization(self): """Test that the expected changes occur and meta-data are unchanged when handling a multi-realization cube.""" cube = self.cube.copy() cube.coord('realization').points = [1] cubes = iris.cube.CubeList([self.cube, cube]) cube = cubes.merge_cube() expected = cube.data.copy() # Output sea-point populated with data from input sea-point: expected[:, 0, 0] = 0.5 # Output sea-point populated with data from input sea-point: expected[:, 1, 1] = 0.5 # Output land-point populated with data from input land-point: expected[:, 0, 1] = 1. # Output land-point populated with data from input sea-point due to # vicinity-constraint: expected[:, 4, 4] = 1. result = self.plugin.process(cube, self.input_land, self.output_land) self.assertIsInstance(result, Cube) self.assertArrayEqual(result.data, expected) self.assertDictEqual(result.attributes, self.cube.attributes) self.assertEqual(result.name(), self.cube.name()) def test_raises_gridding_error(self): """Test error raised when cube and output grids don't match.""" self.cube = self.input_land_ll msg = "X and Y coordinates do not match for cubes" with self.assertRaisesRegex(ValueError, msg): self.plugin.process(self.cube, self.input_land, self.output_land)
class Test_correct_where_input_true(IrisTest): """Tests the correct_where_input_true method of the RegridLandSea class.""" def setUp(self): """Create a class-object containing the necessary cubes. All cubes are on the target grid. Here this is defined as a 3x3 grid. The grid contains ones everywhere except the centre point (a zero). The output_cube has a value of 0.5 at [0, 1]. The move_sea_point cube has the zero value at [0, 1] instead of [1, 1], this allows it to be used in place of input_land to trigger the expected behaviour in the function. """ self.plugin = RegridLandSea(vicinity_radius=2200.) cube = squeeze( set_up_cube(num_grid_points=3, zero_point_indices=((0, 0, 1, 1), ))) self.plugin.input_land = cube.copy() self.plugin.output_land = cube.copy() self.plugin.nearest_cube = cube.copy() self.plugin.nearest_cube.data[0, 1] = 0.5 self.plugin.output_cube = self.plugin.nearest_cube.copy() self.move_sea_point = squeeze( set_up_cube(num_grid_points=3, zero_point_indices=((0, 0, 0, 1), ))) def test_basic_sea(self): """Test that nothing changes with argument zero (sea).""" input_land = self.plugin.input_land.copy() output_land = self.plugin.output_land.copy() output_cube = self.plugin.output_cube.copy() self.plugin.correct_where_input_true(0) self.assertArrayEqual(input_land.data, self.plugin.input_land.data) self.assertArrayEqual(output_land.data, self.plugin.output_land.data) self.assertArrayEqual(output_cube.data, self.plugin.output_cube.data) def test_basic_land(self): """Test that nothing changes with argument one (land).""" input_land = self.plugin.input_land.copy() output_land = self.plugin.output_land.copy() output_cube = self.plugin.output_cube.copy() self.plugin.correct_where_input_true(1) self.assertArrayEqual(input_land.data, self.plugin.input_land.data) self.assertArrayEqual(output_land.data, self.plugin.output_land.data) self.assertArrayEqual(output_cube.data, self.plugin.output_cube.data) def test_work_sea(self): """Test for expected change with argument zero (sea).""" self.plugin.input_land = self.move_sea_point output_cube = self.plugin.output_cube.copy() # The output sea point should have been changed to the value from the # input sea point in the same grid. output_cube.data[1, 1] = 0.5 self.plugin.correct_where_input_true(0) self.assertArrayEqual(output_cube.data, self.plugin.output_cube.data) def test_work_land(self): """Test for expected change with argument one (land).""" self.plugin.input_land = self.move_sea_point output_cube = self.plugin.output_cube.copy() # The input sea point should have been changed to the value from an # input land point in the same grid. output_cube.data[0, 1] = 1.0 self.plugin.correct_where_input_true(1) self.assertArrayEqual(output_cube.data, self.plugin.output_cube.data) def test_work_sealand_eq_landsea(self): """Test result is independent of order of sea/land handling.""" self.plugin.input_land = self.move_sea_point.copy() reset_cube = self.plugin.output_cube.copy() self.plugin.correct_where_input_true(0) self.plugin.correct_where_input_true(1) attempt_01 = self.plugin.output_cube.data.copy() self.plugin.output_cube = reset_cube self.plugin.correct_where_input_true(1) self.plugin.correct_where_input_true(0) attempt_10 = self.plugin.output_cube.data.copy() self.assertArrayEqual(attempt_01, attempt_10) def test_not_in_vicinity(self): """Test for no change if the matching point is too far away.""" # We need larger arrays for this. # Define 5 x 5 arrays with output sea point at [1, 1] and input sea # point at [4, 4]. The alternative value of 0.5 at [4, 4] should not # be selected with a small vicinity_radius. self.plugin = RegridLandSea(vicinity_radius=2200.) cube = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 1, 1), ))) self.plugin.output_land = cube.copy() self.plugin.nearest_cube = cube.copy() self.plugin.nearest_cube.data[4, 4] = 0.5 self.plugin.output_cube = self.plugin.nearest_cube.copy() self.plugin.input_land = squeeze( set_up_cube(num_grid_points=5, zero_point_indices=((0, 0, 4, 4), ))) output_cube = self.plugin.output_cube.copy() self.plugin.correct_where_input_true(0) self.assertArrayEqual(output_cube.data, self.plugin.output_cube.data) def test_no_matching_points(self): """Test code runs and makes no changes if no sea points are present.""" self.plugin.input_land.data = np.ones_like(self.plugin.input_land.data) self.plugin.output_land.data = np.ones_like( self.plugin.output_land.data) output_cube = self.plugin.output_cube.copy() self.plugin.correct_where_input_true(0) self.assertArrayEqual(output_cube.data, self.plugin.output_cube.data) def test_all_matching_points(self): """Test code runs and makes no changes if all land points are present.""" self.plugin.input_land.data = np.ones_like(self.plugin.input_land.data) self.plugin.output_land.data = np.ones_like( self.plugin.output_land.data) output_cube = self.plugin.output_cube.copy() self.plugin.correct_where_input_true(1) self.assertArrayEqual(output_cube.data, self.plugin.output_cube.data)
def test_basic(self): """Test that the expected string is returned.""" expected = ("<RegridLandSea: regridder: Nearest('nanmask'); " "vicinity: <OccurrenceWithinVicinity: distance: 25000.0>>") result = repr(RegridLandSea()) self.assertEqual(result, expected)