def latitude_to_threshold( latitude: np.ndarray, midlatitude: float, tropics: float, ) -> np.ndarray: """ Rescale a latitude range into a range of threshold values suitable for thresholding a different diagnostic. This is based on the value provided for that diagnostic at midlatitude (more than 50 degrees from the equator) and in the tropics (closer than 10 degrees from the equator). Varies linearly in between. Args: latitude: An array of latitude points (e.g. cube.coord("latitude").points) midlatitude: The threshold value to return above 50N or below 50S. tropics: The threshold value to return below 10N or above 10S. Returns: An array of thresholds, one for each latitude point """ return np.where( latitude > 0, rescale(latitude, (50.0, 10), (midlatitude, tropics), clip=True), rescale(latitude, (-50.0, -10), (midlatitude, tropics), clip=True), )
def apply_ice(self, prob_lightning_cube, ice_cube): """ Modify Nowcast of lightning probability with ice data from a radar composite (VII; Vertically Integrated Ice) Args: prob_lightning_cube (iris.cube.Cube): First-guess lightning probability. The forecast_period coord is modified in-place to "minutes". ice_cube (iris.cube.Cube): Analysis of vertically integrated ice (VII) from radar thresholded at self.ice_thresholds. Units of threshold coord modified in-place to kg m^-2 Returns: iris.cube.Cube: Output cube containing updated nowcast lightning probability. This cube will have the same dimensions and meta-data as prob_lightning_cube. The influence of the data in ice_cube reduces linearly to zero as forecast_period increases to 2H30M. Raises: iris.exceptions.ConstraintMismatchError: If ice_cube does not contain the expected thresholds. """ prob_lightning_cube.coord("forecast_period").convert_units("minutes") # check prob-ice threshold units are as expected ice_threshold_coord = find_threshold_coordinate(ice_cube) ice_threshold_coord.convert_units("kg m^-2") new_cube_list = iris.cube.CubeList([]) err_string = "No matching prob(Ice) cube for threshold {}" for cube_slice in prob_lightning_cube.slices_over("time"): fcmins = cube_slice.coord("forecast_period").points[0] for threshold, prob_max in zip(self.ice_thresholds, self.ice_scaling): ice_slice = ice_cube.extract( iris.Constraint(coord_values={ ice_threshold_coord: lambda t: isclose(t.point, threshold) })) if not isinstance(ice_slice, iris.cube.Cube): raise ConstraintMismatchError(err_string.format(threshold)) # Linearly reduce impact of ice as fcmins increases to 2H30M. ice_scaling = [0.0, (prob_max * (1.0 - (fcmins / 150.0)))] if ice_scaling[1] > 0: cube_slice.data = np.maximum( rescale( ice_slice.data, data_range=(0.0, 1.0), scale_range=ice_scaling, clip=True, ), cube_slice.data, ) new_cube_list.append(cube_slice) new_cube = new_cube_list.merge_cube() new_cube = check_cube_coordinates(prob_lightning_cube, new_cube) return new_cube
def test_clip(self): """Test that the method clips values when out of range""" expected = self.cube.data.copy() expected[...] = 108. expected[0, 7, 7] = 100. result = rescale(self.cube.data, data_range=(0.2, 1.2), scale_range=(100., 110.), clip=True) self.assertArrayAlmostEqual(result, expected)
def test_rescaling_outrange(self): """Test that the method gives the expected values when out of range""" expected = self.cube.data.copy() expected[...] = 108. expected[0, 7, 7] = 98. result = rescale(self.cube.data, data_range=(0.2, 1.2), scale_range=(100., 110.)) self.assertArrayAlmostEqual(result, expected)
def test_rescaling_inrange(self): """Test that the method returns the expected values when in range""" expected = self.cube.data.copy() expected[...] = 110. expected[0, 7, 7] = 100. result = rescale(self.cube.data, data_range=(0., 1.), scale_range=(100., 110.)) self.assertArrayAlmostEqual(result, expected)
def test_inverted_clip(self): """Test that the method clips values when the range minimum is not first""" expected = self.cube.data.copy() expected[...] = 108.0 expected[0, 7, 7] = 100.0 result = rescale(self.cube.data, data_range=(1.2, 0.2), scale_range=(110.0, 100.0), clip=True) self.assertArrayAlmostEqual(result, expected)
def _rescale_masked_weights(self, weights): """Apply fuzzy smoothing to weights at the edge of masked areas Args: weights (iris.cube.Cube): Pre-normalised weights where the weights of masked data points have been set to 0 Returns: (tuple): tuple containing: **weights** (iris.cube.Cube): Weights where MASKED slices have been rescaled, but UNMASKED slices have not **rescaled** (iris.cube.Cube): Binary (0/1) map showing which weights have been rescaled """ is_rescaled = iris.cube.CubeList() rescaled_weights = iris.cube.CubeList() for weights_slice in weights.slices_over(self.blend_coord): weights_nonzero = np.where(weights_slice.data > 0, True, False) if np.all(weights_nonzero): # if there are no masked points in this slice, keep current weights # and mark as unchanged (not rescaled) rescaled_weights.append(weights_slice) is_rescaled.append(weights_slice.copy(data=~weights_nonzero)) else: weights_orig = weights_slice.data.copy() # calculate the distance to the nearest invalid point, in grid squares, # for each point on the grid distance = distance_transform_edt(weights_nonzero) # calculate a 0-1 scaling factor based on the distance from the # nearest invalid data point, which scales between 1 at the fuzzy length # towards 0 for points closest to the edge of the mask fuzzy_factor = rescale(distance, data_range=[0.0, self.fuzzy_length], clip=True) # multiply existing weights by fuzzy scaling factor rescaled_weights_data = np.multiply( weights_slice.data, fuzzy_factor).astype(FLOAT_DTYPE) rescaled_weights.append( weights_slice.copy(data=rescaled_weights_data)) # identify spatial points where weights have been rescaled is_rescaled_data = np.where( rescaled_weights_data != weights_orig, True, False) is_rescaled.append(weights_slice.copy(data=is_rescaled_data)) weights = rescaled_weights.merge_cube() rescaled = is_rescaled.merge_cube() return weights, rescaled
def smooth_initial_weights(self, weights_from_mask): """ Create fuzzy weights around points in the weights_from_mask with zero weight. This works by doing an euclidean distance transform based on how far each grid point is from a masked point. This returns an array containing the distance each point is from the nearest masked point. The result is then rescaled so that any point that are at least as far as the fuzzy_length away from a masked point are set back to a weight of one and any points that are closer than the fuzzy_length to a masked point are scaled to be between 0 and 1. Args: weights_from_mask (iris.cube.Cube): A cube containing an initial set of weights based on the mask on the input cube. Returns: result (iris.cube.Cube): A cube containing the fuzzy weights calculated based on the weights_from_mask. The dimension order may have changed from the input cube as it has been sliced over x and y coordinates. """ result = iris.cube.CubeList() x_coord = weights_from_mask.coord(axis='x').name() y_coord = weights_from_mask.coord(axis='y').name() # The distance_transform_edt works on N-D cubes, so we want to make # sure we only apply it to x-y slices. for weights in weights_from_mask.slices([y_coord, x_coord]): if np.all(weights.data == 1.0): # distance_transform_edt doesn't produce what we want if there # are no zeros present. result.append(weights.copy()) else: fuzzy_data = distance_transform_edt(weights.data == 1., 1) fuzzy_data = fuzzy_data.astype(np.float32) rescaled_fuzzy_data = rescale( fuzzy_data, data_range=[0., self.fuzzy_length], clip=True) result.append(weights.copy(data=rescaled_fuzzy_data)) result = result.merge_cube() return result
def process(self, input_cube): """Convert each point to a truth value based on provided threshold values. The truth value may or may not be fuzzy depending upon if fuzzy_bounds are supplied. If the plugin has a "threshold_units" member, this is used to convert both thresholds and fuzzy bounds into the units of the input cube. Args: input_cube (iris.cube.Cube): Cube to threshold. The code is dimension-agnostic. Returns: iris.cube.Cube: Cube after a threshold has been applied. The data within this cube will contain values between 0 and 1 to indicate whether a given threshold has been exceeded or not. The cube meta-data will contain: * Input_cube name prepended with probability_of_X_above(or below)_threshold (where X is the diagnostic under consideration) * Threshold dimension coordinate with same units as input_cube * Threshold attribute ("greater_than", "greater_than_or_equal_to", "less_than", or less_than_or_equal_to" depending on the operator) * Cube units set to (1). Raises: ValueError: if a np.nan value is detected within the input cube. """ if np.isnan(input_cube.data).any(): raise ValueError("Error: NaN detected in input cube data") if self.threshold_units is not None: self.thresholds = [ self.threshold_units.convert(threshold, input_cube.units) for threshold in self.thresholds ] self.fuzzy_bounds = [ tuple([ self.threshold_units.convert(threshold, input_cube.units) for threshold in bounds ]) for bounds in self.fuzzy_bounds ] self.threshold_coord_name = input_cube.name() thresholded_cubes = iris.cube.CubeList() for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds): cube = input_cube.copy() # if upper and lower bounds are equal, set a deterministic 0/1 # probability based on exceedance of the threshold if bounds[0] == bounds[1]: truth_value = self.comparison_operator["function"](cube.data, threshold) # otherwise, scale exceedance probabilities linearly between 0/1 # at the min/max fuzzy bounds and 0.5 at the threshold value else: truth_value = np.where( cube.data < threshold, rescale( cube.data, data_range=(bounds[0], threshold), scale_range=(0.0, 0.5), clip=True, ), rescale( cube.data, data_range=(threshold, bounds[1]), scale_range=(0.5, 1.0), clip=True, ), ) # if requirement is for probabilities less_than or # less_than_or_equal_to the threshold (rather than # greater_than or greater_than_or_equal_to), invert # the exceedance probability if "less_than" in self.comparison_operator["spp_string"]: truth_value = 1.0 - truth_value truth_value = truth_value.astype(FLOAT_DTYPE) if np.ma.is_masked(cube.data): # update unmasked points only cube.data[~input_cube.data.mask] = truth_value[~input_cube. data.mask] else: cube.data = truth_value self._add_threshold_coord(cube, threshold) for func in self.each_threshold_func: cube = func(cube) thresholded_cubes.append(cube) (cube, ) = thresholded_cubes.merge() self._update_metadata(cube) enforce_coordinate_ordering(cube, ["realization", "percentile"]) return cube
def process(self, input_cube): """Convert each point to a truth value based on provided threshold values. The truth value may or may not be fuzzy depending upon if fuzzy_bounds are supplied. Args: input_cube (iris.cube.Cube): Cube to threshold. The code is dimension-agnostic. Returns: cube (iris.cube.Cube): Cube after a threshold has been applied. The data within this cube will contain values between 0 and 1 to indicate whether a given threshold has been exceeded or not. The cube meta-data will contain: * input_cube name prepended with `probability_of_` * threshold dimension coordinate with same units as input_cube * threshold attribute (above or below threshold) * cube units set to (1). Raises: ValueError: if a np.nan value is detected within the input cube. """ thresholded_cubes = iris.cube.CubeList() if np.isnan(input_cube.data).any(): raise ValueError("Error: NaN detected in input cube data") for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds): cube = input_cube.copy() if bounds[0] == bounds[1]: truth_value = cube.data > threshold else: truth_value = np.where( cube.data < threshold, rescale(cube.data, data_range=(bounds[0], threshold), scale_range=(0., 0.5), clip=True), rescale(cube.data, data_range=(threshold, bounds[1]), scale_range=(0.5, 1.), clip=True), ) truth_value = truth_value.astype(np.float64) if self.below_thresh_ok: truth_value = 1. - truth_value cube.data = truth_value coord = iris.coords.DimCoord(threshold, long_name="threshold", units=cube.units) cube.add_aux_coord(coord) cube = iris.util.new_axis(cube, 'threshold') thresholded_cubes.append(cube) cube, = thresholded_cubes.concatenate() # TODO: Correct when formal cf-standards exists # Force the metadata to temporary conventions if self.below_thresh_ok: cube.attributes.update({'relative_to_threshold': 'below'}) else: cube.attributes.update({'relative_to_threshold': 'above'}) cube.rename("probability_of_{}".format(cube.name())) cube.units = Unit(1) cube = ExtractData.make_stat_coordinate_first(cube) return cube
def test_zero_range_output(self): """Test that the method returns the expected error""" msg = "Cannot rescale a zero output range" with self.assertRaisesRegex(ValueError, msg): rescale(self.cube.data, scale_range=[4, 4])
def test_basic(self): """Test that the method returns the expected array type""" result = rescale(self.cube.data) self.assertIsInstance(result, np.ndarray)
def process(self, input_cube): """Convert each point to a truth value based on provided threshold values. The truth value may or may not be fuzzy depending upon if fuzzy_bounds are supplied. If the plugin has a "threshold_units" member, this is used to convert both thresholds and fuzzy bounds into the units of the input cube. Args: input_cube (iris.cube.Cube): Cube to threshold. The code is dimension-agnostic. Returns: cube (iris.cube.Cube): Cube after a threshold has been applied. The data within this cube will contain values between 0 and 1 to indicate whether a given threshold has been exceeded or not. The cube meta-data will contain: * Input_cube name prepended with probability_of_X_above(or below)_threshold (where X is the diagnostic under consideration) * Threshold dimension coordinate with same units as input_cube * Threshold attribute (above or below threshold) * Cube units set to (1). Raises: ValueError: if a np.nan value is detected within the input cube. """ # Record input cube data type to ensure consistent output, though # integer data must become float to enable fuzzy thresholding. input_cube_dtype = input_cube.dtype if input_cube.dtype.kind == 'i': input_cube_dtype = np.float32 thresholded_cubes = iris.cube.CubeList() if np.isnan(input_cube.data).any(): raise ValueError("Error: NaN detected in input cube data") # if necessary, convert thresholds and fuzzy bounds into cube units if self.threshold_units is not None: self.thresholds = [ self.threshold_units.convert(threshold, input_cube.units) for threshold in self.thresholds ] self.fuzzy_bounds = [ tuple([ self.threshold_units.convert(threshold, input_cube.units) for threshold in bounds ]) for bounds in self.fuzzy_bounds ] # set name of threshold coordinate to match input diagnostic self.threshold_coord_name = input_cube.name() # apply fuzzy thresholding for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds): cube = input_cube.copy() # if upper and lower bounds are equal, set a deterministic 0/1 # probability based on exceedance of the threshold if bounds[0] == bounds[1]: truth_value = cube.data > threshold # otherwise, scale exceedance probabilities linearly between 0/1 # at the min/max fuzzy bounds and 0.5 at the threshold value else: truth_value = np.where( cube.data < threshold, rescale(cube.data, data_range=(bounds[0], threshold), scale_range=(0., 0.5), clip=True), rescale(cube.data, data_range=(threshold, bounds[1]), scale_range=(0.5, 1.), clip=True), ) truth_value = truth_value.astype(input_cube_dtype) # if requirement is for probabilities below threshold (rather than # above), invert the exceedance probability if self.below_thresh_ok: truth_value = 1. - truth_value cube.data = truth_value # Overwrite masked values that have been thresholded # with the un-thresholded values from the input cube. if np.ma.is_masked(cube.data): cube.data[input_cube.data.mask] = ( input_cube.data[input_cube.data.mask]) cube = self._add_threshold_coord(cube, threshold) thresholded_cubes.append(cube) cube, = thresholded_cubes.concatenate() # TODO: Correct when formal cf-standards exists # Force the metadata to temporary conventions if self.below_thresh_ok: cube.attributes.update({'relative_to_threshold': 'below'}) cube.rename("probability_of_{}_below_threshold".format( cube.name())) else: cube.attributes.update({'relative_to_threshold': 'above'}) cube.rename("probability_of_{}_above_threshold".format( cube.name())) cube.units = Unit(1) cube = enforce_coordinate_ordering(cube, ["realization", "percentile_over"]) return cube
def process(self, input_cube): """Convert each point to a truth value based on provided threshold values. The truth value may or may not be fuzzy depending upon if fuzzy_bounds are supplied. If the plugin has a "threshold_units" member, this is used to convert both thresholds and fuzzy bounds into the units of the input cube. Args: input_cube (iris.cube.Cube): Cube to threshold. The code is dimension-agnostic. Returns: iris.cube.Cube: Cube after a threshold has been applied. The data within this cube will contain values between 0 and 1 to indicate whether a given threshold has been exceeded or not. The cube meta-data will contain: * Input_cube name prepended with probability_of_X_above(or below)_threshold (where X is the diagnostic under consideration) * Threshold dimension coordinate with same units as input_cube * Threshold attribute ("greater_than", "greater_than_or_equal_to", "less_than", or less_than_or_equal_to" depending on the operator) * Cube units set to (1). Raises: ValueError: if a np.nan value is detected within the input cube. """ # Record input cube data type to ensure consistent output, though # integer data must become float to enable fuzzy thresholding. input_cube_dtype = input_cube.dtype if input_cube.dtype.kind == "i": input_cube_dtype = FLOAT_DTYPE thresholded_cubes = iris.cube.CubeList() if np.isnan(input_cube.data).any(): raise ValueError("Error: NaN detected in input cube data") # if necessary, convert thresholds and fuzzy bounds into cube units if self.threshold_units is not None: self.thresholds = [ self.threshold_units.convert(threshold, input_cube.units) for threshold in self.thresholds ] self.fuzzy_bounds = [ tuple([ self.threshold_units.convert(threshold, input_cube.units) for threshold in bounds ]) for bounds in self.fuzzy_bounds ] # set name of threshold coordinate to match input diagnostic self.threshold_coord_name = input_cube.name() # apply fuzzy thresholding for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds): cube = input_cube.copy() # if upper and lower bounds are equal, set a deterministic 0/1 # probability based on exceedance of the threshold if bounds[0] == bounds[1]: truth_value = self.comparison_operator["function"](cube.data, threshold) # otherwise, scale exceedance probabilities linearly between 0/1 # at the min/max fuzzy bounds and 0.5 at the threshold value else: truth_value = np.where( cube.data < threshold, rescale( cube.data, data_range=(bounds[0], threshold), scale_range=(0.0, 0.5), clip=True, ), rescale( cube.data, data_range=(threshold, bounds[1]), scale_range=(0.5, 1.0), clip=True, ), ) # if requirement is for probabilities less_than or # less_than_or_equal_to the threshold (rather than # greater_than or greater_than_or_equal_to), invert # the exceedance probability if "less_than" in self.comparison_operator["spp_string"]: truth_value = 1.0 - truth_value truth_value = np.ma.masked_where(np.ma.getmask(cube.data), truth_value) truth_value = truth_value.astype(input_cube_dtype) cube.data = truth_value # Overwrite masked values that have been thresholded # with the un-thresholded values from the input cube. if np.ma.is_masked(cube.data): cube.data[input_cube.data.mask] = input_cube.data[ input_cube.data.mask] cube = self._add_threshold_coord(cube, threshold) for func in self.each_threshold_func: cube = func(cube) thresholded_cubes.append(cube) (cube, ) = thresholded_cubes.concatenate() if len(self.thresholds) == 1: # if only one threshold has been provided, this should be scalar cube = next(cube.slices_over(cube.coord(var_name="threshold"))) cube.rename("probability_of_{}_{}_threshold".format( cube.name(), probability_is_above_or_below(cube))) cube.units = Unit(1) enforce_coordinate_ordering(cube, ["realization", "percentile"]) return cube