def test_defaults(self): """Test default arguments produce cube with expected dimensions and metadata""" result = set_up_variable_cube(self.data) # check type, data and attributes self.assertIsInstance(result, iris.cube.Cube) self.assertEqual(result.standard_name, "air_temperature") self.assertEqual(result.name(), "air_temperature") self.assertEqual(result.units, "K") self.assertArrayAlmostEqual(result.data, self.data) self.assertEqual(result.attributes, {}) # check dimension coordinates self.assertEqual(result.coord_dims("latitude"), (0, )) self.assertEqual(result.coord_dims("longitude"), (1, )) # check scalar time coordinates for time_coord in ["time", "forecast_reference_time"]: self.assertEqual(result.coord(time_coord).dtype, np.int64) self.assertEqual(result.coord("forecast_period").dtype, np.int32) expected_time = datetime(2017, 11, 10, 4, 0) time_point = iris_time_to_datetime(result.coord("time"))[0] self.assertEqual(time_point, expected_time) expected_frt = datetime(2017, 11, 10, 0, 0) frt_point = iris_time_to_datetime( result.coord("forecast_reference_time"))[0] self.assertEqual(frt_point, expected_frt) self.assertEqual(result.coord("forecast_period").units, "seconds") self.assertEqual(result.coord("forecast_period").points[0], 14400) check_mandatory_standards(result)
def test_float64_cube_data(self): """Test a failure of a cube with 64-bit float data.""" self.cube.data = self.cube.data.astype(np.float64) msg = ("does not have required dtype.\n" "Expected: float32, Actual: float64") with self.assertRaisesRegex(ValueError, msg): check_mandatory_standards(self.cube)
def test_defaults(self): """Test default arguments produce cube with expected dimensions and metadata""" result = set_up_percentile_cube(self.data, self.percentiles) perc_coord = result.coord("percentile") self.assertArrayEqual(perc_coord.points, self.percentiles) self.assertEqual(perc_coord.units, "%") check_mandatory_standards(result)
def test_float64_cube_coord_points(self): """Test a failure of a cube with 64-bit float coord points.""" self.cube.coord("projection_x_coordinate").points = self.cube.coord( "projection_x_coordinate").points.astype(np.float64) msg = ("does not have required dtype.\n" "Expected: float32, Actual \\(points\\): float64") with self.assertRaisesRegex(ValueError, msg): check_mandatory_standards(self.cube)
def assert_metadata_ok(output_cube): """Checks that the meta-data of output_cube are as expected""" assert isinstance(output_cube, Cube) assert output_cube.dtype == np.float32 assert list(output_cube.coord_dims("time")) == [ n for n, in [output_cube.coord_dims(c) for c in ["latitude", "longitude"]] ] assert output_cube.coord("time").dtype == np.int64 check_mandatory_standards(output_cube)
def test_float64_cube_coord_bounds(self): """Test a failure of a cube with 64-bit float coord bounds.""" x_coord = self.cube.coord("projection_x_coordinate") x_coord.bounds = np.array([(point - 10.0, point + 10.0) for point in x_coord.points], dtype=np.float64) msg = ("does not have required dtype.\n" "Expected: float32, " "Actual \\(points\\): float32, " "Actual \\(bounds\\): float64") with self.assertRaisesRegex(ValueError, msg): check_mandatory_standards(self.cube)
def test_basic(self): """Test addition of a leading height coordinate""" result = add_coordinate( self.input_cube, self.height_points, 'height', coord_units=self.height_unit) self.assertIsInstance(result, iris.cube.Cube) self.assertSequenceEqual(result.shape, (10, 3, 4)) self.assertEqual(result.coord_dims('height'), (0,)) self.assertArrayAlmostEqual( result.coord('height').points, self.height_points) self.assertEqual(result.coord('height').dtype, np.float32) self.assertEqual(result.coord('height').units, self.height_unit) check_mandatory_standards(result)
def forecast_period_coord( cube: Cube, force_lead_time_calculation: bool = False ) -> Coord: """ Return the lead time coordinate (forecast_period) from a cube, either by reading an existing forecast_period coordinate, or by calculating the difference between time and forecast_reference_time. Args: cube: Cube from which the lead times will be determined. force_lead_time_calculation: Force the lead time to be calculated from the forecast_reference_time and the time coordinate, even if the forecast_period coordinate exists. Default is False. Returns: New forecast_period coord. A DimCoord is returned if the forecast_period coord is already present in the cube as a DimCoord and this coord does not need changing, otherwise it will be an AuxCoord. """ create_dim_coord = False if cube.coords("forecast_period"): if isinstance(cube.coord("forecast_period"), DimCoord): create_dim_coord = True if cube.coords("forecast_period") and not force_lead_time_calculation: result_coord = cube.coord("forecast_period").copy() elif cube.coords("time") and cube.coords("forecast_reference_time"): # Cube must adhere to mandatory standards for safe time calculations check_mandatory_standards(cube) # Try to calculate forecast period from forecast reference time and # time coordinates result_coord = _calculate_forecast_period( cube.coord("time"), cube.coord("forecast_reference_time"), dim_coord=create_dim_coord, ) else: msg = ( "The forecast period coordinate is not available within {}." "The time coordinate and forecast_reference_time " "coordinate were also not available for calculating " "the forecast_period.".format(cube) ) raise CoordinateNotFoundError(msg) return result_coord
def test_conformant_cubes(self): """Test conformant data, percentile and probability cubes all pass (no error is thrown and cube is not changed)""" cubelist = [ self.cube, self.probability_cube, self.percentile_cube] for cube in cubelist: result = cube.copy() check_mandatory_standards(result) # The following statement renders each cube into an XML string # describing all aspects of the cube (including a checksum of the # data) to verify that nothing has been changed anywhere on the # cube. self.assertStringEqual(CubeList([cube]).xml(checksum=True), CubeList([result]).xml(checksum=True))
def test_defaults(self): """Test default arguments produce cube with expected dimensions and metadata""" result = set_up_probability_cube(self.data, self.thresholds) thresh_coord = find_threshold_coordinate(result) self.assertEqual(result.name(), "probability_of_air_temperature_above_threshold") self.assertEqual(result.units, "1") self.assertArrayEqual(thresh_coord.points, self.thresholds) self.assertEqual(thresh_coord.name(), "air_temperature") self.assertEqual(thresh_coord.var_name, "threshold") self.assertEqual(thresh_coord.units, "K") self.assertEqual(len(thresh_coord.attributes), 1) self.assertEqual(thresh_coord.attributes["spp__relative_to_threshold"], "above") check_mandatory_standards(result)
def _check_metadata(cube): """ Checks cube metadata that needs to be correct to guarantee data integrity Args: cube (iris.cube.Cube): Cube to be checked Raises: ValueError: if time coordinates do not have the required datatypes and units; needed because values may be wrong ValueError: if numerical datatypes are other than 32-bit (except where specified); needed because values may be wrong ValueError: if cube dataset has unknown units; because this may cause misinterpretation on "load" """ check_mandatory_standards(cube) if cf_units.Unit(cube.units).is_unknown(): raise ValueError("{} has unknown units".format(cube.name()))
def test_multiple_errors(self): """Test a list of errors is correctly caught and re-raised""" self.percentile_cube.coord( "percentile").points = self.percentile_cube.coord( "percentile").points.astype(np.float64) self.percentile_cube.coord("forecast_period").convert_units("minutes") self.percentile_cube.coord( "forecast_period").points = self.percentile_cube.coord( "forecast_period").points.astype(np.int64) msg = ("percentile of type .*DimCoord.* " "does not have required dtype.\n" "Expected: float32, Actual \\(points\\): float64\n" "forecast_period of type .*DimCoord.* " "does not have required dtype.\n" "Expected: int32, Actual \\(points\\): int64\n" "forecast_period of type .*DimCoord.* " "does not have required units.\n" "Expected: seconds, Actual: minutes") with self.assertRaisesRegex(ValueError, msg): check_mandatory_standards(self.percentile_cube)
def run(self, cube: Cube) -> None: """Populates self-consistent interpreted parameters, or raises collated errors describing (as far as posible) how the metadata are a) not self-consistent, and / or b) not consistent with the Met Office IMPROVER standard. Although every effort has been made to return as much information as possible, collated errors may not be complete if the issue is fundamental. The developer is advised to rerun this tool after each fix, until no further problems are raised. """ # 1) Interpret diagnostic and type-specific metadata, including cell methods if cube.name() in ANCILLARIES: self.field_type = self.ANCIL self.diagnostic = cube.name() if cube.cell_methods: self.errors.append(f"Unexpected cell methods {cube.cell_methods}") elif cube.name() in SPECIAL_CASES: self.field_type = self.diagnostic = cube.name() if cube.name() == "weather_code": for cm in cube.cell_methods: if cm == WXCODE_MODE_CM and cube.name() in WXCODE_NAMES: pass else: self.errors.append( f"Unexpected cell methods {cube.cell_methods}" ) elif cube.name() == "wind_from_direction": if cube.cell_methods: expected = CellMethod(method="mean", coords="realization") if len(cube.cell_methods) > 1 or cube.cell_methods[0] != expected: self.errors.append( f"Unexpected cell methods {cube.cell_methods}" ) else: self.unhandled = True return else: if "probability" in cube.name() and "threshold" in cube.name(): self.field_type = self.PROB self.check_probability_cube_metadata(cube) else: self.diagnostic = cube.name() try: perc_coord = find_percentile_coordinate(cube) except CoordinateNotFoundError: coords = get_coord_names(cube) if any( [cube.coord(coord).var_name == "threshold" for coord in coords] ): self.field_type = self.PROB self.check_probability_cube_metadata(cube) else: self.field_type = self.DIAG else: self.field_type = self.PERC if perc_coord.name() != PERC_COORD: self.errors.append( f"Percentile coordinate should have name {PERC_COORD}, " f"has {perc_coord.name()}" ) if perc_coord.units != "%": self.errors.append( "Percentile coordinate should have units of %, " f"has {perc_coord.units}" ) self.check_cell_methods(cube) # 2) Interpret model and blend information from cube attributes self.check_attributes(cube.attributes) # 3) Check whether expected coordinates are present coords = get_coord_names(cube) if "spot_index" in coords: self.check_spot_data(cube, coords) if self.field_type == self.ANCIL: # there is no definitive standard for time coordinates on static ancillaries pass elif cube.coords("time_in_local_timezone"): # For data on local timezones, the time coordinate will match the horizontal # dimensions and there will be no forecast period. expected_coords = set(LOCAL_TIME_COORDS + UNBLENDED_TIME_COORDS) expected_coords.discard("forecast_period") self._check_coords_present(coords, expected_coords) self._check_coords_are_horizontal(cube, ["time"]) elif self.blended: self._check_coords_present(coords, BLENDED_TIME_COORDS) else: self._check_coords_present(coords, UNBLENDED_TIME_COORDS) # 4) Check points are equal to upper bounds for bounded time coordinates for coord in ["time", "forecast_period"]: if coord in get_coord_names(cube): self._check_coord_bounds(cube, coord) # 5) Check datatypes on data and coordinates try: check_mandatory_standards(cube) except ValueError as cause: self.errors.append(str(cause)) # 6) Check multiple realizations only exist for ensemble models if self.field_type == self.DIAG: try: realization_coord = cube.coord("realization") except CoordinateNotFoundError: pass else: model_id = cube.attributes.get(self.model_id_attr, "ens") if "ens" not in model_id and len(realization_coord.points) > 1: self.errors.append( f"Deterministic model should not have {len(realization_coord.points)} " "realizations" ) # 7) Raise collated errors if present if self.errors: raise ValueError("\n".join(self.errors))
def set_up_variable_cube(data, name='air_temperature', units='K', spatial_grid='latlon', time=datetime(2017, 11, 10, 4, 0), time_bounds=None, frt=datetime(2017, 11, 10, 0, 0), realizations=None, include_scalar_coords=None, attributes=None, standard_grid_metadata=None): """ Set up a cube containing a single variable field with: - x/y spatial dimensions (equal area or lat / lon) - optional leading "realization" dimension - "time", "forecast_reference_time" and "forecast_period" scalar coords - option to specify additional scalar coordinates - configurable attributes Args: data (numpy.ndarray): 2D (y-x ordered) or 3D (realization-y-x ordered) array of data to put into the cube. name (str): Variable name (standard / long) units (str): Variable units spatial_grid (str): What type of x/y coordinate values to use. Permitted values are "latlon" or "equalarea". time (datetime.datetime): Single cube validity time time_bounds (tuple or list of datetime.datetime instances): Lower and upper bound on time point, if required frt (datetime.datetime): Single cube forecast reference time realizations (list or numpy.ndarray): List of forecast realizations. If not present, taken from the leading dimension of the input data array (if 3D). include_scalar_coords (list): List of iris.coords.DimCoord or AuxCoord instances of length 1. attributes (dict): Optional cube attributes. standard_grid_metadata (str): Recognised mosg__model_configuration for which to set up Met Office standard grid attributes. Should be 'uk_det', 'uk_ens', 'gl_det' or 'gl_ens'. """ # construct spatial dimension coordimates ypoints = data.shape[-2] xpoints = data.shape[-1] y_coord, x_coord = construct_xy_coords(ypoints, xpoints, spatial_grid) # construct realization dimension for 3D data, and dim_coords list ndims = len(data.shape) if ndims == 3: if realizations is not None: if len(realizations) != data.shape[0]: raise ValueError( 'Cannot generate {} realizations from data of shape ' '{}'.format(len(realizations), data.shape)) realizations = np.array(realizations) if issubclass(realizations.dtype.type, np.integer): # expect integer realizations realizations = realizations.astype(np.int32) else: # option needed for percentile & probability cube setup realizations = realizations.astype(np.float32) else: realizations = np.arange(data.shape[0]).astype(np.int32) realization_coord = DimCoord(realizations, "realization", units="1") dim_coords = [(realization_coord, 0), (y_coord, 1), (x_coord, 2)] elif ndims == 2: dim_coords = [(y_coord, 0), (x_coord, 1)] else: raise ValueError( 'Expected 2 or 3 dimensions on input data: got {}'.format(ndims)) # construct list of aux_coords_and_dims scalar_coords = construct_scalar_time_coords(time, time_bounds, frt) if include_scalar_coords is not None: for coord in include_scalar_coords: scalar_coords.append((coord, None)) # set up attributes cube_attrs = {} if standard_grid_metadata is not None: cube_attrs.update(MOSG_GRID_DEFINITION[standard_grid_metadata]) if attributes is not None: cube_attrs.update(attributes) # create data cube cube = iris.cube.Cube(data, units=units, attributes=cube_attrs, dim_coords_and_dims=dim_coords, aux_coords_and_dims=scalar_coords) cube.rename(name) # don't allow unit tests to set up invalid cubes check_mandatory_standards(cube) return cube
def test_string_coord(self): """Test conformant data with a cube with a coord of strings.""" self.cube.add_aux_coord(AuxCoord(["kittens"], long_name="animal")) check_mandatory_standards(self.cube)
def set_up_variable_cube( data, name="air_temperature", units="K", spatial_grid="latlon", time=datetime(2017, 11, 10, 4, 0), time_bounds=None, frt=datetime(2017, 11, 10, 0, 0), realizations=None, include_scalar_coords=None, attributes=None, standard_grid_metadata=None, grid_spacing=None, domain_corner=None, height_levels=None, pressure=False, ): """ Set up a cube containing a single variable field with: - x/y spatial dimensions (equal area or lat / lon) - optional leading "realization" dimension - optional "height" dimension - "time", "forecast_reference_time" and "forecast_period" scalar coords - option to specify additional scalar coordinates - configurable attributes Args: data (numpy.ndarray): 2D (y-x ordered) or 3D (realization-y-x ordered) array of data to put into the cube. name (Optional[str]): Variable name (standard / long) units (Optional[str]): Variable units spatial_grid (Optional[str]): What type of x/y coordinate values to use. Permitted values are "latlon" or "equalarea". time (Optional[datetime.datetime]): Single cube validity time time_bounds (Optional[Sequence[datetime.datetime]]): Lower and upper bound on time point, if required frt (Optional[datetime.datetime]): Single cube forecast reference time realizations (Optional[List[numpy.ndarray]]): List of forecast realizations. If not present, taken from the leading dimension of the input data array (if 3D). include_scalar_coords (Optional[List[iris.coords.DimCoord] or List[iris.coords.AuxCoord]]): List of iris.coords.DimCoord or AuxCoord instances of length 1. attributes (Optional[Dict[Any]]): Optional cube attributes. standard_grid_metadata (Optional[str]): Recognised mosg__model_configuration for which to set up Met Office standard grid attributes. Should be 'uk_det', 'uk_ens', 'gl_det' or 'gl_ens'. grid_spacing (Optional[float]): Grid resolution (degrees for latlon or metres for equalarea). domain_corner (Optional[Tuple[float, float]]): Bottom left corner of grid domain (y,x) (degrees for latlon or metres for equalarea). height_levels (Optional[List[float]]): List of height levels in metres or pressure levels in Pa. pressure (Optional[bool]): Flag to indicate whether the height levels are specified as pressure, in Pa. If False, use height in metres. Returns: iris.cube.Cube: Cube containing a single variable field """ # construct spatial dimension coordimates ypoints = data.shape[-2] xpoints = data.shape[-1] y_coord, x_coord = construct_yx_coords( ypoints, xpoints, spatial_grid, grid_spacing=grid_spacing, domain_corner=domain_corner, ) dim_coords = _construct_dimension_coords( data, y_coord, x_coord, realizations, height_levels, pressure ) # construct list of aux_coords_and_dims scalar_coords = construct_scalar_time_coords(time, time_bounds, frt) if include_scalar_coords is not None: for coord in include_scalar_coords: scalar_coords.append((coord, None)) # set up attributes cube_attrs = {} if standard_grid_metadata is not None: cube_attrs.update(MOSG_GRID_DEFINITION[standard_grid_metadata]) if attributes is not None: cube_attrs.update(attributes) # create data cube cube = iris.cube.Cube( data, units=units, attributes=cube_attrs, dim_coords_and_dims=dim_coords, aux_coords_and_dims=scalar_coords, ) cube.rename(name) # don't allow unit tests to set up invalid cubes check_mandatory_standards(cube) return cube
def test_int64_cube_data(self): """Test conformant data with a cube with 64-bit integer data.""" self.cube.data = self.cube.data.astype(np.int64) check_mandatory_standards(self.cube)
def _check_inputs(self, cubes): """Check the inputs prior to calculating the accumulations. Args: cubes: iris.cube.CubeList Cube list of precipitation rates that will be checked for their appropriateness in calculating the requested accumulations. The timesteps between the cubes in this cubelist are expected to be regular. Returns: (tuple): tuple containing: **cubes** (iris.cube.CubeList): Modified version of the input cube list of precipitation rates that have had the units of the coordinates and cube data enforced. The cube list has also been sorted by time. **time_interval** (float): Interval between the timesteps from the input cubelist. Raises: ValueError: The input rates cubes must be at regularly spaced time intervals. ValueError: The accumulation period is less than the time interval between the rates cubes. ValueError: The specified accumulation period is not cleanly divisible by the time interval. """ # Standardise inputs to expected units standardised_cubes = [] for cube in cubes: check_mandatory_standards(cube) new_cube = cube.copy() new_cube.convert_units("m s-1") standardised_cubes.append(new_cube) cubes = standardised_cubes # Sort cubes into time order and calculate intervals. cubes, times = self.sort_cubes_by_time(cubes) try: (time_interval, ) = np.unique(np.diff(times, axis=0)).astype(np.int32) except ValueError: msg = ("Accumulation is designed to work with " "rates cubes at regular time intervals. Cubes " "provided are unevenly spaced in time; time intervals are " "{}.".format(np.diff(times, axis=0))) raise ValueError(msg) if self.accumulation_period is None: # If no accumulation period is specified, assume that the input # cubes will be used to construct a single accumulation period. (self.accumulation_period, ) = cubes[-1].coord("forecast_period").points # Ensure that the accumulation period is int32. self.accumulation_period = np.int32(self.accumulation_period) fraction, integral = np.modf(self.accumulation_period / time_interval) # Check whether the accumulation period is less than the time_interval # i.e. the integral is equal to zero. In this case, the rates cubes # are too widely spaced to compute the requested accumulation period. if integral == 0: msg = ( "The accumulation_period is less than the time interval " "between the rates cubes. The rates cubes provided are " "therefore insufficient for computing the accumulation period " "requested. accumulation period specified: {}, " "time interval specified: {}".format(self.accumulation_period, time_interval)) raise ValueError(msg) # Ensure the accumulation period is cleanly divisible by the time # interval. if fraction != 0: msg = ("The specified accumulation period ({}) is not divisible " "by the time intervals between rates cubes ({}). As " "a result it is not possible to calculate the desired " "total accumulation period.".format( self.accumulation_period, time_interval)) raise ValueError(msg) if self.forecast_periods is None: # If no forecast periods are specified, then the accumulation # periods calculated will end at the forecast period from # each of the input cubes. self.forecast_periods = [ cube.coord("forecast_period").points for cube in cubes if cube.coord("forecast_period").points >= time_interval ] # Check whether any forecast periods are less than the accumulation # period. This is expected if the accumulation period is e.g. 1 hour, # however, the forecast periods are e.g. [15, 30, 45] minutes. # In this case, the forecast periods are filtered, so that only # complete accumulation periods will be calculated. if any(self.forecast_periods < self.accumulation_period): forecast_periods = [ fp for fp in self.forecast_periods if fp >= self.accumulation_period ] self.forecast_periods = forecast_periods return cubes, time_interval