def _add_iris_coord(cube, name, points, dim, calendar=None): """ Add a Coord to a Cube from a Pandas index or columns array. If no calendar is specified for a time series, Gregorian is assumed. """ units = Unit("unknown") if calendar is None: calendar = cf_units.CALENDAR_GREGORIAN # Convert pandas datetime objects to python datetime obejcts. if isinstance(points, DatetimeIndex): points = np.array([i.to_datetime() for i in points]) # Convert datetime objects to Iris' current datetime representation. if points.dtype == object: dt_types = (datetime.datetime, netcdftime.datetime) if all([isinstance(i, dt_types) for i in points]): units = Unit("hours since epoch", calendar=calendar) points = units.date2num(points) points = np.array(points) if (np.issubdtype(points.dtype, np.number) and iris.util.monotonic(points, strict=True)): coord = DimCoord(points, units=units) coord.rename(name) cube.add_dim_coord(coord, dim) else: coord = AuxCoord(points, units=units) coord.rename(name) cube.add_aux_coord(coord, dim)
def units_convertible(units1, units2, reftimeistime=True): """Return True if a Unit representing the string units1 can be converted to a Unit representing the string units2, else False.""" try: u1 = Unit(units1) u2 = Unit(units2) except ValueError: return False return u1.is_convertible(units2)
def _convert_coord_unit_to_datetime(coord, dt): """Converts a datetime to be in the unit of a specified Coord. """ if isinstance(coord, iris.coords.Coord): # The unit class is then cf_units.Unit. iris_unit = coord.units else: iris_unit = Unit(coord.units) return iris_unit.num2date(dt)
def unit_converter(data, inunit, outunit): ''' Unit converter. Takes an (numpy) array, valid udunits inunits and outunits as strings, and returns the array in outunits. Parameters ---------- data : array_like inunit : string unit to convert from, must be UDUNITS-compatible string outunit : string unit to conver to, must be UDUNITS-compatible string Returns ------- out : array_like Example ------- >>> import numpy as np >>> c = Converter("kg","Gt") >>> out = c(np.array([1,2])*1e12) >>> out = array([ 1., 2.]) ''' inunit = str(inunit) outunit = str(outunit) if isinstance(data, np.ma.MaskedArray): mask = data.mask else: mask = None data = np.array(data) if not (inunit == outunit): try: try: from cf_units import Unit in_unit = Unit(inunit) out_unit = Unit(outunit) outdata = in_unit.convert(data, out_unit) except: from udunits2 import Converter, System, Unit sys = System() c = Converter((Unit(sys, inunit), Unit(sys, outunit))) outdata = c(data) except: print( "Neither cf_units or udunits2 module found, you're on your own.") c = 1. / 1e3 outdata = c * data else: outdata = data if mask is not None: return np.ma.array(outdata, mask=mask) else: return outdata
def test_360_day_calendar(self): n = 360 calendar = '360_day' time_unit = Unit('days since 1970-01-01 00:00', calendar=calendar) time_coord = AuxCoord(np.arange(n), 'time', units=time_unit) expected_ydata = np.array([CalendarDateTime(time_unit.num2date(point), calendar) for point in time_coord.points]) line1, = iplt.plot(time_coord) result_ydata = line1.get_ydata() self.assertArrayEqual(expected_ydata, result_ydata)
def test_gregorian_calendar_conversion_setup(self): # Reproduces a situation where a unit's gregorian calendar would not # match (using the `is` operator) to the literal string 'gregorian', # causing an `is not` test to return a false negative. cal_str = cf_units.CALENDAR_GREGORIAN calendar = self.MyStr(cal_str) self.assertIsNot(calendar, cal_str) u1 = Unit('hours since 1970-01-01 00:00:00', calendar=calendar) u2 = Unit('hours since 1969-11-30 00:00:00', calendar=calendar) u1point = np.array([8.], dtype=np.float32) expected = np.array([776.], dtype=np.float32) result = u1.convert(u1point, u2) return expected, result
def test_360_day_calendar(self): n = 360 calendar = '360_day' time_unit = Unit('days since 1970-01-01 00:00', calendar=calendar) time_coord = AuxCoord(np.arange(n), 'time', units=time_unit) times = [time_unit.num2date(point) for point in time_coord.points] times = [netcdftime.datetime(atime.year, atime.month, atime.day, atime.hour, atime.minute, atime.second) for atime in times] expected_ydata = np.array([CalendarDateTime(time, calendar) for time in times]) line1, = iplt.plot(time_coord) result_ydata = line1.get_ydata() self.assertArrayEqual(expected_ydata, result_ydata)
def __init__(self): self.u = Unit('days since 1600-01-01 00:00:00', calendar=CALENDAR_STANDARD) self.points = np.arange(1, 5, 1) self.coord = iris.coords.DimCoord(self.points, units=self.u) self.start = datetime.datetime(2000, 1, 1) self.end = datetime.datetime(2003, 4, 24) self.start = self.u.date2num(self.start) self.end = self.u.date2num(self.end)
def setUp(self): self.section = {'year': 2007, 'month': 1, 'day': 15, 'hour': 0, 'minute': 3, 'second': 0} self.unit = Unit('hours since epoch', calendar=CALENDAR_GREGORIAN) dt = datetime(self.section['year'], self.section['month'], self.section['day'], self.section['hour'], self.section['minute'], self.section['second']) self.point = self.unit.date2num(dt)
def setUp(self): self.section = {"year": 2007, "month": 1, "day": 15, "hour": 0, "minute": 3, "second": 0} self.unit = Unit("hours since epoch", calendar=CALENDAR_GREGORIAN) dt = datetime( self.section["year"], self.section["month"], self.section["day"], self.section["hour"], self.section["minute"], self.section["second"], ) self.point = self.unit.date2num(dt)
class Test(tests.IrisTest): def setUp(self): self.section = {'year': 2007, 'month': 1, 'day': 15, 'hour': 0, 'minute': 3, 'second': 0} self.unit = Unit('hours since epoch', calendar=CALENDAR_GREGORIAN) dt = datetime(self.section['year'], self.section['month'], self.section['day'], self.section['hour'], self.section['minute'], self.section['second']) self.point = self.unit.date2num(dt) def _check(self, section, standard_name=None): expected = DimCoord(self.point, standard_name=standard_name, units=self.unit) # The call being tested. coord = reference_time_coord(section) self.assertEqual(coord, expected) def test_start_of_forecast_0(self): section = deepcopy(self.section) section['significanceOfReferenceTime'] = 0 self._check(section, 'forecast_reference_time') def test_start_of_forecast_1(self): section = deepcopy(self.section) section['significanceOfReferenceTime'] = 1 self._check(section, 'forecast_reference_time') def test_observation_time(self): section = deepcopy(self.section) section['significanceOfReferenceTime'] = 3 self._check(section, 'time') def test_unknown_significance(self): section = deepcopy(self.section) section['significanceOfReferenceTime'] = 5 emsg = 'unsupported significance' with self.assertRaisesRegexp(TranslationError, emsg): self._check(section)
class Test(tests.IrisTest): def setUp(self): self.section = {"year": 2007, "month": 1, "day": 15, "hour": 0, "minute": 3, "second": 0} self.unit = Unit("hours since epoch", calendar=CALENDAR_GREGORIAN) dt = datetime( self.section["year"], self.section["month"], self.section["day"], self.section["hour"], self.section["minute"], self.section["second"], ) self.point = self.unit.date2num(dt) def _check(self, section, standard_name=None): expected = DimCoord(self.point, standard_name=standard_name, units=self.unit) # The call being tested. coord = reference_time_coord(section) self.assertEqual(coord, expected) def test_start_of_forecast(self): section = deepcopy(self.section) section["significanceOfReferenceTime"] = 1 self._check(section, "forecast_reference_time") def test_observation_time(self): section = deepcopy(self.section) section["significanceOfReferenceTime"] = 3 self._check(section, "time") def test_unknown_significance(self): section = deepcopy(self.section) section["significanceOfReferenceTime"] = 0 emsg = "unsupported significance" with self.assertRaisesRegexp(TranslationError, emsg): self._check(section)
def units(self, u): """Units of the data""" self._optinfo['units'] = u if isinstance(u, Unit) else Unit(u)
def process(self, input_cube): """Convert each point to a truth value based on provided threshold values. The truth value may or may not be fuzzy depending upon if fuzzy_bounds are supplied. If the plugin has a "threshold_units" member, this is used to convert both thresholds and fuzzy bounds into the units of the input cube. Args: input_cube (iris.cube.Cube): Cube to threshold. The code is dimension-agnostic. Returns: cube (iris.cube.Cube): Cube after a threshold has been applied. The data within this cube will contain values between 0 and 1 to indicate whether a given threshold has been exceeded or not. The cube meta-data will contain: * input_cube name prepended with `probability_of_` * threshold dimension coordinate with same units as input_cube * threshold attribute (above or below threshold) * cube units set to (1). Raises: ValueError: if a np.nan value is detected within the input cube. """ # Record input cube data type to ensure consistent output, though # integer data must become float to enable fuzzy thresholding. input_cube_dtype = input_cube.dtype if input_cube.dtype.kind == 'i': input_cube_dtype = np.float32 thresholded_cubes = iris.cube.CubeList() if np.isnan(input_cube.data).any(): raise ValueError("Error: NaN detected in input cube data") # if necessary, convert thresholds and fuzzy bounds into cube units if self.threshold_units is not None: self.thresholds = [ self.threshold_units.convert(threshold, input_cube.units) for threshold in self.thresholds ] self.fuzzy_bounds = [ tuple([ self.threshold_units.convert(threshold, input_cube.units) for threshold in bounds ]) for bounds in self.fuzzy_bounds ] # apply fuzzy thresholding for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds): cube = input_cube.copy() # if upper and lower bounds are equal, set a deterministic 0/1 # probability based on exceedance of the threshold if bounds[0] == bounds[1]: truth_value = cube.data > threshold # otherwise, scale exceedance probabilities linearly between 0/1 # at the min/max fuzzy bounds and 0.5 at the threshold value else: truth_value = np.where( cube.data < threshold, rescale(cube.data, data_range=(bounds[0], threshold), scale_range=(0., 0.5), clip=True), rescale(cube.data, data_range=(threshold, bounds[1]), scale_range=(0.5, 1.), clip=True), ) truth_value = truth_value.astype(input_cube_dtype) # if requirement is for probabilities below threshold (rather than # above), invert the exceedance probability if self.below_thresh_ok: truth_value = 1. - truth_value cube.data = truth_value coord = iris.coords.DimCoord(threshold, long_name="threshold", units=cube.units) cube.add_aux_coord(coord) cube = iris.util.new_axis(cube, 'threshold') thresholded_cubes.append(cube) cube, = thresholded_cubes.concatenate() # TODO: Correct when formal cf-standards exists # Force the metadata to temporary conventions if self.below_thresh_ok: cube.attributes.update({'relative_to_threshold': 'below'}) else: cube.attributes.update({'relative_to_threshold': 'above'}) cube.rename("probability_of_{}".format(cube.name())) cube.units = Unit(1) cube = enforce_coordinate_ordering(cube, ["realization", "percentile_over"]) return cube
def __init__(self, thresholds, fuzzy_factor=None, fuzzy_bounds=None, threshold_units=None, below_thresh_ok=False): """ Set up for processing an in-or-out of threshold field, including the generation of fuzzy_bounds which are required to threshold an input cube (through self.process(cube)). If fuzzy_factor is not None, fuzzy bounds are calculated using the threshold value in the units in which it is provided. The usage of fuzzy_factor is exemplified as follows: For a 6 mm/hr threshold with a 0.75 fuzzy factor, a range of 25% around this threshold (between (6*0.75=) 4.5 and (6*(2-0.75)=) 7.5) would be generated. The probabilities of exceeding values within this range are scaled linearly, so that 4.5 mm/hr yields a thresholded value of 0 and 7.5 mm/hr yields a thresholded value of 1. Therefore, in this case, the thresholded exceedance probabilities between 4.5 mm/hr and 7.5 mm/hr would follow the pattern: :: Data value | Probability ------------|------------- 4.5 | 0 5.0 | 0.167 5.5 | 0.333 6.0 | 0.5 6.5 | 0.667 7.0 | 0.833 7.5 | 1.0 Args: thresholds (list of floats or float): The threshold points for 'significant' datapoints. Keyword Args: fuzzy_factor (float): Specifies lower bound for fuzzy membership value when multiplied by each threshold. Upper bound is equivalent linear distance above threshold. If None, no fuzzy_factor is applied. fuzzy_bounds (list of tuples): Lower and upper bounds for fuzziness. List should be of same length as thresholds. Each entry in list should be a tuple of two floats representing the lower and upper bounds respectively. If None, no fuzzy_bounds are applied. threshold_units (string): Units of the threshold values. If not provided the units are assumed to be the same as those of the input cube. below_thresh_ok (boolean): True to count points as significant if *below* the threshold, False to count points as significant if *above* the threshold. Raises: ValueError: If a threshold of 0.0 is requested when using a fuzzy factor. ValueError: If the fuzzy_factor is not greater than 0 and less than 1. ValueError: If both fuzzy_factor and fuzzy_bounds are set as this is ambiguous. """ # ensure threshold is a list, even if only a single value is provided self.thresholds = thresholds if np.isscalar(thresholds): self.thresholds = [thresholds] # if necessary, set threshold units if threshold_units is None: self.threshold_units = None else: self.threshold_units = Unit(threshold_units) # initialise threshold coordinate name as None self.threshold_coord_name = None # read fuzzy factor or set (default) to 1 (no smoothing) fuzzy_factor_loc = 1. if fuzzy_factor is not None: if fuzzy_bounds is not None: raise ValueError( "Invalid combination of keywords. Cannot specify " "fuzzy_factor and fuzzy_bounds together") if not 0 < fuzzy_factor < 1: raise ValueError( "Invalid fuzzy_factor: must be >0 and <1: {}".format( fuzzy_factor)) if 0 in self.thresholds: raise ValueError( "Invalid threshold with fuzzy factor: cannot use a " "multiplicative fuzzy factor with threshold == 0") fuzzy_factor_loc = fuzzy_factor # Set fuzzy-bounds. If neither fuzzy_factor nor fuzzy_bounds is set, # both lower_thr and upper_thr default to the threshold value. A test # of this equality is used later to determine whether to process with # a sharp threshold or fuzzy bounds. if fuzzy_bounds is None: self.fuzzy_bounds = [] for thr in self.thresholds: lower_thr = thr * fuzzy_factor_loc upper_thr = thr * (2. - fuzzy_factor_loc) if thr < 0: lower_thr, upper_thr = upper_thr, lower_thr self.fuzzy_bounds.append((lower_thr, upper_thr)) else: self.fuzzy_bounds = fuzzy_bounds # ensure fuzzy_bounds is a list of tuples if isinstance(fuzzy_bounds, tuple): self.fuzzy_bounds = [fuzzy_bounds] # check that thresholds and fuzzy_bounds are self-consistent for thr, bounds in zip(self.thresholds, self.fuzzy_bounds): assert len(bounds) == 2, ("Invalid bounds for one threshold: {}. " "Expected 2 floats.".format(bounds)) bounds_msg = ("Threshold must be within bounds: " "!( {} <= {} <= {} )".format(bounds[0], thr, bounds[1])) assert bounds[0] <= thr, bounds_msg assert bounds[1] >= thr, bounds_msg self.below_thresh_ok = below_thresh_ok
def test_fix_data(self): cube = self.fix.fix_data(self.cube) self.assertEqual(cube.data[0], 12.0 / 44.0) self.assertEqual(cube.units, Unit('J'))
def units_temporal(units): u = Unit(units) return u.is_time_reference()
def add_coordinate( incube: Cube, coord_points: List, coord_name: str, coord_units: Optional[str] = None, dtype: Type = np.float32, order: Optional[List[int]] = None, is_datetime: bool = False, attributes: Optional[Dict[str, Any]] = None, ) -> Cube: """ Function to duplicate a sample cube with an additional coordinate to create a cubelist. The cubelist is merged to create a single cube, which can be reordered to place the new coordinate in the required position. Args: incube: Cube to be duplicated. coord_points: Values for the coordinate. coord_name: Long name of the coordinate to be added. coord_units: Coordinate unit required. dtype: Datatype for coordinate points. order: Optional list of integers to reorder the dimensions on the new merged cube. For example, if the new coordinate is required to be in position 1 on a 4D cube, use order=[1, 0, 2, 3] to swap the new coordinate position with that of the original leading coordinate. is_datetime: If "true", the leading coordinate points have been given as a list of datetime objects and need converting. In this case the "coord_units" argument is overridden and the time points provided in seconds. The "dtype" argument is overridden and set to int64. attributes: Optional coordinate attributes. Returns: Cube containing an additional dimension coordinate. """ # if the coordinate already exists as a scalar coordinate, remove it cube = incube.copy() try: cube.remove_coord(coord_name) except CoordinateNotFoundError: pass # if new coordinate points are provided as datetimes, convert to seconds if is_datetime: coord_units = TIME_COORDS["time"].units dtype = TIME_COORDS["time"].dtype new_coord_points = [_create_time_point(val) for val in coord_points] coord_points = new_coord_points cubes = iris.cube.CubeList([]) for val in coord_points: temp_cube = cube.copy() temp_cube.add_aux_coord( DimCoord( np.array([val], dtype=dtype), long_name=coord_name, units=coord_units, attributes=attributes, )) # recalculate forecast period if time or frt have been updated if (coord_name in ["time", "forecast_reference_time"] and coord_units is not None and Unit(coord_units).is_time_reference()): forecast_period = forecast_period_coord( temp_cube, force_lead_time_calculation=True) try: temp_cube.replace_coord(forecast_period) except CoordinateNotFoundError: temp_cube.add_aux_coord(forecast_period) cubes.append(temp_cube) new_cube = cubes.merge_cube() if order is not None: new_cube.transpose(order) return new_cube
def set_up_probability_cube( data: ndarray, thresholds: Union[List[float], ndarray], variable_name: str = "air_temperature", threshold_units: str = "K", spp__relative_to_threshold: str = "greater_than", **kwargs: Any, ) -> Cube: """ Set up a cube containing probabilities at thresholds with: - x/y spatial dimensions (equal area or lat / lon) - leading "threshold" dimension - "time", "forecast_reference_time" and "forecast_period" scalar coords - option to specify additional scalar coordinates - "spp__relative_to_threshold" attribute (default "greater_than") - default or configurable attributes - configurable cube data, name conforms to "probability_of_X_above(or below)_threshold" convention Args: data: 3D (threshold-y-x ordered) array of data to put into the cube thresholds: List of int / float threshold values whose length must match the first dimension on the input data cube variable_name: Name of the underlying variable to which the probability field applies, eg "air_temperature". NOT name of probability field. threshold_units: Units of the underlying variable / threshold. spp__relative_to_threshold: Value of the attribute "spp__relative_to_threshold" which is required for IMPROVER probability cubes. **kwargs: Additional keyword arguments passed to 'set_up_variable_cube' function Returns: Cube containing probabilities at thresholds """ # create a "relative to threshold" attribute coord_attributes = { "spp__relative_to_threshold": spp__relative_to_threshold } if spp__relative_to_threshold in ( "above", "greater_than", "greater_than_or_equal_to", ): name = "probability_of_{}_above_threshold".format(variable_name) elif spp__relative_to_threshold in ("below", "less_than", "less_than_or_equal_to"): name = "probability_of_{}_below_threshold".format(variable_name) else: msg = ("The spp__relative_to_threshold attribute MUST be set for " "IMPROVER probability cubes") raise ValueError(msg) cube = set_up_variable_cube( data, name=name, units="1", realizations=thresholds, **kwargs, ) threshold_name = variable_name.replace("_in_vicinity", "") cube.coord("realization").rename(threshold_name) cube.coord(threshold_name).var_name = "threshold" cube.coord(threshold_name).attributes.update(coord_attributes) cube.coord(threshold_name).units = Unit(threshold_units) if len(thresholds) == 1: cube = next(cube.slices_over(threshold_name)) return cube
def main(argv=None): """Load in arguments and ensure they are set correctly. Then run Triangular weighted blending across the given coordinate.""" parser = ArgParser( description='Use the TriangularWeightedBlendAcrossAdjacentPoints to ' 'blend across a particular coordinate. It does not ' 'collapse the coordinate, but instead blends across ' 'adjacent points and puts the blended values back in the ' 'original coordinate, with adjusted bounds.') parser.add_argument('coordinate', type=str, metavar='COORDINATE_TO_BLEND_OVER', help='The coordinate over which the blending ' 'will be applied.') parser.add_argument('central_point', metavar='CENTRAL_POINT', type=float, help='Central point at which the output from the ' 'triangular weighted blending will be ' 'calculated. This should be in the units of the ' 'units argument that is passed in. ' 'This value should be a point on the ' 'coordinate for blending over.') parser.add_argument('--units', metavar='UNIT_STRING', required=True, help='Units of the the central_point and width.') parser.add_argument('--calendar', metavar='CALENDAR', default='gregorian', help='Calendar for parameter_unit if required. ' 'Default=gregorian') parser.add_argument('--width', metavar='TRIANGLE_WIDTH', type=float, required=True, help='Width of the triangular weighting function used ' 'in the blending, in the units of the ' 'units argument passed in.') parser.add_argument('--blend_time_using_forecast_period', default=False, action='store_true', help='Flag that ' 'we are blending over time but using the forecast ' 'period coordinate as a proxy. Note this should only ' 'be used when time and forecast_period share a ' 'dimension: ie when all files provided are from the ' 'same forecast cycle.') parser.add_argument('input_filepaths', metavar='INPUT_FILES', nargs="+", help='Paths to input NetCDF files including and ' 'surrounding the central_point.') parser.add_argument('output_filepath', metavar='OUTPUT_FILE', help='The output path for the processed NetCDF.') args = parser.parse_args(args=argv) # TriangularWeightedBlendAcrossAdjacentPoints can't currently handle # blending over times where iris reads the coordinate points as datetime # objects. Fail here to avoid unhelpful errors downstream. if "time" in args.coordinate: msg = ("Cannot blend over {} coordinate (points encoded as datetime " "objects)".format(args.coordinate)) raise ValueError(msg) # This is left as a placeholder for when we have this capability if args.coordinate == 'time': units = Unit(args.units, args.calendar) else: units = args.units cubelist = load_cubelist(args.input_filepaths) if (args.blend_time_using_forecast_period and args.coordinate == 'forecast_period'): cube = MergeCubes().process(cubelist, check_time_bounds_ranges=True) elif args.blend_time_using_forecast_period: msg = ('"--blend_time_using_forecast_period" can only be used with ' '"forecast_period" coordinate') raise ValueError(msg) else: cube = MergeCubes().process(cubelist) BlendingPlugin = TriangularWeightedBlendAcrossAdjacentPoints( args.coordinate, args.central_point, units, args.width) result = BlendingPlugin.process(cube) save_netcdf(result, args.output_filepath)
def test_eta_incompatible_units(self): self.eta.units = Unit("km") with self.assertRaises(ValueError): OceanSigmaFactory(**self.kwargs)
def test_eta_incompatible_units(self): new_eta = mock.Mock(units=Unit("Pa"), nbounds=0) with self.assertRaises(ValueError): self.factory.update(self.eta, new_eta)
def process(cubelist, coordinate, central_point, units, width, calendar='gregorian', blend_time_using_forecast_period=False): """Runs weighted blending across adjacent points. Uses the TriangularWeightedBlendAcrossAdjacentPoints to blend across a particular coordinate. It does not collapse the coordinate, but instead blends across adjacent points and puts the blended values back in the original coordinate, with adjusted bounds. Args: cubelist (iris.cube.CubeList): CubeList including and surrounding the central point. coordinate (str): The coordinate over which the blending will be applied. central_point (float): Central point at which the output from the triangular weighted blending will be calculated. This should be in the units of the units argument that is passed in. This value should be a point on the coordinate for blending over. units (str): Units of the central_point and width width (float): Width of the triangular weighting function used in the blending, in the units of the units argument. calendar (str) Calendar for parameter_unit if required. Default is 'gregorian'. blend_time_using_forecast_period (bool): If True, we are blending over time but using the forecast period coordinate as a proxy. Note, this should only be used when time and forecast_period share a dimension: i.e when all cubes provided are from the same forecast cycle. Default is False. Returns: iris.cube.Cube: A processed Cube Raises: ValueError: If coordinate has "time" in it. ValueError: If blend_time_forecast_period is not used with forecast_period coordinate. """ # TriangularWeightedBlendAcrossAdjacentPoints can't currently handle # blending over times where iris reads the coordinate points as datetime # objects. Fail here to avoid unhelpful errors downstream. if "time" in coordinate: msg = ("Cannot blend over {} coordinate (points encoded as datetime " "objects)".format(coordinate)) raise ValueError(msg) # This is left as a placeholder for when we have this capability if coordinate == 'time': units = Unit(units, calendar) if blend_time_using_forecast_period and coordinate == 'forecast_period': cube = MergeCubes().process(cubelist, check_time_bounds_ranges=True) elif blend_time_using_forecast_period: msg = ('"--blend_time_using_forecast_period" can only be used with ' '"forecast_period" coordinate') raise ValueError(msg) else: cube = MergeCubes().process(cubelist) blending_plugin = TriangularWeightedBlendAcrossAdjacentPoints( coordinate, central_point, units, width) result = blending_plugin.process(cube) return result
def units_temporal(units): try: u = Unit(units) except ValueError: return False return u.is_time_reference()
def __init__(self, thresholds, fuzzy_factor=None, fuzzy_bounds=None, threshold_units=None, comparison_operator='>'): """ Set up for processing an in-or-out of threshold field, including the generation of fuzzy_bounds which are required to threshold an input cube (through self.process(cube)). If fuzzy_factor is not None, fuzzy bounds are calculated using the threshold value in the units in which it is provided. The usage of fuzzy_factor is exemplified as follows: For a 6 mm/hr threshold with a 0.75 fuzzy factor, a range of 25% around this threshold (between (6*0.75=) 4.5 and (6*(2-0.75)=) 7.5) would be generated. The probabilities of exceeding values within this range are scaled linearly, so that 4.5 mm/hr yields a thresholded value of 0 and 7.5 mm/hr yields a thresholded value of 1. Therefore, in this case, the thresholded exceedance probabilities between 4.5 mm/hr and 7.5 mm/hr would follow the pattern: :: Data value | Probability ------------|------------- 4.5 | 0 5.0 | 0.167 5.5 | 0.333 6.0 | 0.5 6.5 | 0.667 7.0 | 0.833 7.5 | 1.0 Args: thresholds (list of float or float): The threshold points for 'significant' datapoints. fuzzy_factor (float): Specifies lower bound for fuzzy membership value when multiplied by each threshold. Upper bound is equivalent linear distance above threshold. If None, no fuzzy_factor is applied. fuzzy_bounds (list of tuple): Lower and upper bounds for fuzziness. List should be of same length as thresholds. Each entry in list should be a tuple of two floats representing the lower and upper bounds respectively. If None, no fuzzy_bounds are applied. threshold_units (str): Units of the threshold values. If not provided the units are assumed to be the same as those of the input cube. comparison_operator (str): Indicates the comparison_operator to use with the threshold. e.g. 'ge' or '>=' to evaluate data >= threshold or '<' to evaluate data < threshold. When using fuzzy thresholds, there is no difference between < and <= or > and >=. Valid choices: > >= < <= gt ge lt le. Raises: ValueError: If a threshold of 0.0 is requested when using a fuzzy factor. ValueError: If the fuzzy_factor is not greater than 0 and less than 1. ValueError: If both fuzzy_factor and fuzzy_bounds are set as this is ambiguous. """ # ensure threshold is a list, even if only a single value is provided self.thresholds = thresholds if np.isscalar(thresholds): self.thresholds = [thresholds] # if necessary, set threshold units if threshold_units is None: self.threshold_units = None else: self.threshold_units = Unit(threshold_units) # initialise threshold coordinate name as None self.threshold_coord_name = None # read fuzzy factor or set (default) to 1 (no smoothing) fuzzy_factor_loc = 1. if fuzzy_factor is not None: if fuzzy_bounds is not None: raise ValueError( "Invalid combination of keywords. Cannot specify " "fuzzy_factor and fuzzy_bounds together") if not 0 < fuzzy_factor < 1: raise ValueError( "Invalid fuzzy_factor: must be >0 and <1: {}".format( fuzzy_factor)) if 0 in self.thresholds: raise ValueError( "Invalid threshold with fuzzy factor: cannot use a " "multiplicative fuzzy factor with threshold == 0") fuzzy_factor_loc = fuzzy_factor # Set fuzzy-bounds. If neither fuzzy_factor nor fuzzy_bounds is set, # both lower_thr and upper_thr default to the threshold value. A test # of this equality is used later to determine whether to process with # a sharp threshold or fuzzy bounds. if fuzzy_bounds is None: self.fuzzy_bounds = [] for thr in self.thresholds: lower_thr = thr * fuzzy_factor_loc upper_thr = thr * (2. - fuzzy_factor_loc) if thr < 0: lower_thr, upper_thr = upper_thr, lower_thr self.fuzzy_bounds.append((lower_thr, upper_thr)) else: self.fuzzy_bounds = fuzzy_bounds # ensure fuzzy_bounds is a list of tuples if isinstance(fuzzy_bounds, tuple): self.fuzzy_bounds = [fuzzy_bounds] # check that thresholds and fuzzy_bounds are self-consistent for thr, bounds in zip(self.thresholds, self.fuzzy_bounds): if len(bounds) != 2: raise ValueError("Invalid bounds for one threshold: {}." " Expected 2 floats.".format(bounds)) if bounds[0] > thr or bounds[1] < thr: bounds_msg = ("Threshold must be within bounds: " "!( {} <= {} <= {} )".format( bounds[0], thr, bounds[1])) raise ValueError(bounds_msg) # Dict of known logical comparisons. Each key contains a dict of # {'function': The operator function for this comparison_operator, # 'spp_string': Comparison_Operator string for use in CF-convention # meta-data} self.comparison_operator_dict = {} self.comparison_operator_dict.update( dict.fromkeys(['ge', 'GE', '>='], { 'function': operator.ge, 'spp_string': 'above' })) self.comparison_operator_dict.update( dict.fromkeys(['gt', 'GT', '>'], { 'function': operator.gt, 'spp_string': 'above' })) self.comparison_operator_dict.update( dict.fromkeys(['le', 'LE', '<='], { 'function': operator.le, 'spp_string': 'below' })) self.comparison_operator_dict.update( dict.fromkeys(['lt', 'LT', '<'], { 'function': operator.lt, 'spp_string': 'below' })) self.comparison_operator_string = comparison_operator self._decode_comparison_operator_string()
def check_ancils( a_over_s_cube: Cube, sigma_cube: Cube, z0_cube: Optional[Cube], pp_oro_cube: Cube, model_oro_cube: Cube, ) -> ndarray: """Check ancils grid and units. Check if ancil cubes are on the same grid and if they have the expected units. The testing for "same grid" might be replaced if there is a general utils function made for it or so. Args: a_over_s_cube: holding the silhouette roughness field sigma_cube: holding the standard deviation of height in a grid cell z0_cube: holding the vegetative roughness field pp_oro_cube: holding the post processing grid orography model_oro_cube: holding the model orography on post processing grid Returns: Containing bools describing whether or not the tests passed """ ancil_list = [a_over_s_cube, sigma_cube, pp_oro_cube, model_oro_cube] unwanted_coord_list = [ "time", "height", "model_level_number", "forecast_time", "forecast_reference_time", "forecast_period", ] for field, exp_unit in zip( ancil_list, [1, Unit("m"), Unit("m"), Unit("m")]): for unwanted_coord in unwanted_coord_list: try: field.remove_coord(unwanted_coord) except CoordinateNotFoundError: pass if field.units != exp_unit: msg = ("{} ancil field has unexpected unit:" " {} (expected) vs. {} (actual)") raise ValueError( msg.format(field.name(), exp_unit, field.units)) if z0_cube is not None: ancil_list.append(z0_cube) for unwanted_coord in unwanted_coord_list: try: z0_cube.remove_coord(unwanted_coord) except CoordinateNotFoundError: pass if z0_cube.units != Unit("m"): msg = "z0 ancil has unexpected unit: should be {} " "is {}" raise ValueError(msg.format(Unit("m"), z0_cube.units)) permutated_ancil_list = list(itertools.permutations(ancil_list, 2)) oklist = [] for entry in permutated_ancil_list: x_axis_flag = entry[0].coord(axis="y") == entry[1].coord(axis="y") y_axis_flag = entry[0].coord(axis="x") == entry[1].coord(axis="x") oklist.append(x_axis_flag & y_axis_flag) # HybridHeightToPhenomOnPressure._cube_compatibility_check(entr[0], # entr[1]) return np.array(oklist).all() # replace by a return value of True
def NAME_to_cube(filenames, callback): """ Returns a generator of cubes given a list of filenames and a callback. """ for filename in filenames: header, column_headings, data_arrays = load_NAME_III(filename) for i, data_array in enumerate(data_arrays): # turn the dictionary of column headers with a list of header # information for each field into a dictionary of headers for just # this field. Ignore the first 4 columns of grid position (data was # located with the data array). field_headings = dict((k, v[i + 4]) for k, v in column_headings.items()) # make an cube cube = iris.cube.Cube(data_array) # define the name and unit name = ('%s %s' % (field_headings['species'], field_headings['quantity'])) name = name.upper().replace(' ', '_') cube.rename(name) # Some units are badly encoded in the file, fix this by putting a # space in between. (if gs is not found, then the string will be # returned unchanged) cube.units = field_headings['unit'].replace('gs', 'g s') # define and add the singular coordinates of the field (flight # level, time etc.) cube.add_aux_coord(icoords.AuxCoord(field_headings['z_level'], long_name='flight_level', units='1')) # define the time unit and use it to serialise the datetime for the # time coordinate time_unit = Unit('hours since epoch', calendar=CALENDAR_GREGORIAN) time_coord = icoords.AuxCoord( time_unit.date2num(field_headings['time']), standard_name='time', units=time_unit) cube.add_aux_coord(time_coord) # build a coordinate system which can be referenced by latitude and # longitude coordinates lat_lon_coord_system = icoord_systems.GeogCS(6371229) # build regular latitude and longitude coordinates which have # bounds start = header['X grid origin'] + header['X grid resolution'] step = header['X grid resolution'] count = header['X grid size'] pts = start + np.arange(count, dtype=np.float32) * step lon_coord = icoords.DimCoord(pts, standard_name='longitude', units='degrees', coord_system=lat_lon_coord_system) lon_coord.guess_bounds() start = header['Y grid origin'] + header['Y grid resolution'] step = header['Y grid resolution'] count = header['Y grid size'] pts = start + np.arange(count, dtype=np.float32) * step lat_coord = icoords.DimCoord(pts, standard_name='latitude', units='degrees', coord_system=lat_lon_coord_system) lat_coord.guess_bounds() # add the latitude and longitude coordinates to the cube, with # mappings to data dimensions cube.add_dim_coord(lat_coord, 0) cube.add_dim_coord(lon_coord, 1) # implement standard iris callback capability. Although callbacks # are not used in this example, the standard mechanism for a custom # loader to implement a callback is shown: cube = iris.io.run_callback(callback, cube, [header, field_headings, data_array], filename) # yield the cube created (the loop will continue when the next() # element is requested) yield cube
def setUp(self): self.sigma = mock.Mock(units=Unit('1'), nbounds=0) self.eta = mock.Mock(units=Unit('m'), nbounds=0) self.depth = mock.Mock(units=Unit('m'), nbounds=0) self.kwargs = dict(sigma=self.sigma, eta=self.eta, depth=self.depth) self.factory = OceanSigmaFactory(**self.kwargs)
class BasicThreshold(object): """Apply a threshold truth criterion to a cube. Calculate the threshold truth values based on a linear membership function around the threshold values provided. A cube will be returned with a new threshold dimension coordinate. Can operate on multiple time sequences within a cube. """ def __init__(self, thresholds, fuzzy_factor=None, fuzzy_bounds=None, threshold_units=None, below_thresh_ok=False): """ Set up for processing an in-or-out of threshold field, including the generation of fuzzy_bounds which are required to threshold an input cube (through self.process(cube)). If fuzzy_factor is not None, fuzzy bounds are calculated using the threshold value in the units in which it is provided. The usage of fuzzy_factor is exemplified as follows: For a 6 mm/hr threshold with a 0.75 fuzzy factor, a range of 25% around this threshold (between (6*0.75=) 4.5 and (6*(2-0.75)=) 7.5) would be generated. The probabilities of exceeding values within this range are scaled linearly, so that 4.5 mm/hr yields a thresholded value of 0 and 7.5 mm/hr yields a thresholded value of 1. Therefore, in this case, the thresholded exceedance probabilities between 4.5 mm/hr and 7.5 mm/hr would follow the pattern: :: Data value | Probability ------------|------------- 4.5 | 0 5.0 | 0.167 5.5 | 0.333 6.0 | 0.5 6.5 | 0.667 7.0 | 0.833 7.5 | 1.0 Args: thresholds (list of floats or float): The threshold points for 'significant' datapoints. Keyword Args: fuzzy_factor (float): Specifies lower bound for fuzzy membership value when multiplied by each threshold. Upper bound is equivalent linear distance above threshold. If None, no fuzzy_factor is applied. fuzzy_bounds (list of tuples): Lower and upper bounds for fuzziness. List should be of same length as thresholds. Each entry in list should be a tuple of two floats representing the lower and upper bounds respectively. If None, no fuzzy_bounds are applied. threshold_units (string): Units of the threshold values. If not provided the units are assumed to be the same as those of the input cube. below_thresh_ok (boolean): True to count points as significant if *below* the threshold, False to count points as significant if *above* the threshold. Raises: ValueError: If a threshold of 0.0 is requested when using a fuzzy factor. ValueError: If the fuzzy_factor is not greater than 0 and less than 1. ValueError: If both fuzzy_factor and fuzzy_bounds are set as this is ambiguous. """ # ensure threshold is a list, even if only a single value is provided self.thresholds = thresholds if np.isscalar(thresholds): self.thresholds = [thresholds] # if necessary, set threshold units if threshold_units is None: self.threshold_units = None else: self.threshold_units = Unit(threshold_units) # initialise threshold coordinate name as None self.threshold_coord_name = None # read fuzzy factor or set (default) to 1 (no smoothing) fuzzy_factor_loc = 1. if fuzzy_factor is not None: if fuzzy_bounds is not None: raise ValueError( "Invalid combination of keywords. Cannot specify " "fuzzy_factor and fuzzy_bounds together") if not 0 < fuzzy_factor < 1: raise ValueError( "Invalid fuzzy_factor: must be >0 and <1: {}".format( fuzzy_factor)) if 0 in self.thresholds: raise ValueError( "Invalid threshold with fuzzy factor: cannot use a " "multiplicative fuzzy factor with threshold == 0") fuzzy_factor_loc = fuzzy_factor # Set fuzzy-bounds. If neither fuzzy_factor nor fuzzy_bounds is set, # both lower_thr and upper_thr default to the threshold value. A test # of this equality is used later to determine whether to process with # a sharp threshold or fuzzy bounds. if fuzzy_bounds is None: self.fuzzy_bounds = [] for thr in self.thresholds: lower_thr = thr * fuzzy_factor_loc upper_thr = thr * (2. - fuzzy_factor_loc) if thr < 0: lower_thr, upper_thr = upper_thr, lower_thr self.fuzzy_bounds.append((lower_thr, upper_thr)) else: self.fuzzy_bounds = fuzzy_bounds # ensure fuzzy_bounds is a list of tuples if isinstance(fuzzy_bounds, tuple): self.fuzzy_bounds = [fuzzy_bounds] # check that thresholds and fuzzy_bounds are self-consistent for thr, bounds in zip(self.thresholds, self.fuzzy_bounds): assert len(bounds) == 2, ("Invalid bounds for one threshold: {}. " "Expected 2 floats.".format(bounds)) bounds_msg = ("Threshold must be within bounds: " "!( {} <= {} <= {} )".format(bounds[0], thr, bounds[1])) assert bounds[0] <= thr, bounds_msg assert bounds[1] >= thr, bounds_msg self.below_thresh_ok = below_thresh_ok def __repr__(self): """Represent the configured plugin instance as a string.""" return ('<BasicThreshold: thresholds {}, ' + 'fuzzy_bounds {}, ' + 'below_thresh_ok: {}>').format(self.thresholds, self.fuzzy_bounds, self.below_thresh_ok) def _add_threshold_coord(self, cube, threshold): """ Add a scalar threshold-type coordinate to a cube containing thresholded data and promote the new coordinate to be the leading dimension of the cube. Args: cube (iris.cube.Cube): Cube containing thresholded data (1s and 0s) threshold (np.float32): Value at which the data has been thresholded Returns: iris.cube.Cube: With new "threshold" axis """ try: coord = iris.coords.DimCoord( np.array([threshold], dtype=np.float32), standard_name=self.threshold_coord_name, var_name="threshold", units=cube.units) except ValueError as cause: if 'is not a valid standard_name' in str(cause): coord = iris.coords.DimCoord( np.array([threshold], dtype=np.float32), long_name=self.threshold_coord_name, var_name="threshold", units=cube.units) else: raise ValueError(cause) # Use an spp__relative_to_threshold attribute, as an extension to the # CF-conventions. if self.below_thresh_ok: coord.attributes.update({'spp__relative_to_threshold': 'below'}) else: coord.attributes.update({'spp__relative_to_threshold': 'above'}) cube.add_aux_coord(coord) return iris.util.new_axis(cube, coord) def process(self, input_cube): """Convert each point to a truth value based on provided threshold values. The truth value may or may not be fuzzy depending upon if fuzzy_bounds are supplied. If the plugin has a "threshold_units" member, this is used to convert both thresholds and fuzzy bounds into the units of the input cube. Args: input_cube (iris.cube.Cube): Cube to threshold. The code is dimension-agnostic. Returns: cube (iris.cube.Cube): Cube after a threshold has been applied. The data within this cube will contain values between 0 and 1 to indicate whether a given threshold has been exceeded or not. The cube meta-data will contain: * Input_cube name prepended with probability_of_X_above(or below)_threshold (where X is the diagnostic under consideration) * Threshold dimension coordinate with same units as input_cube * Threshold attribute (above or below threshold) * Cube units set to (1). Raises: ValueError: if a np.nan value is detected within the input cube. """ # Record input cube data type to ensure consistent output, though # integer data must become float to enable fuzzy thresholding. input_cube_dtype = input_cube.dtype if input_cube.dtype.kind == 'i': input_cube_dtype = np.float32 thresholded_cubes = iris.cube.CubeList() if np.isnan(input_cube.data).any(): raise ValueError("Error: NaN detected in input cube data") # if necessary, convert thresholds and fuzzy bounds into cube units if self.threshold_units is not None: self.thresholds = [ self.threshold_units.convert(threshold, input_cube.units) for threshold in self.thresholds ] self.fuzzy_bounds = [ tuple([ self.threshold_units.convert(threshold, input_cube.units) for threshold in bounds ]) for bounds in self.fuzzy_bounds ] # set name of threshold coordinate to match input diagnostic self.threshold_coord_name = input_cube.name() # apply fuzzy thresholding for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds): cube = input_cube.copy() # if upper and lower bounds are equal, set a deterministic 0/1 # probability based on exceedance of the threshold if bounds[0] == bounds[1]: truth_value = cube.data > threshold # otherwise, scale exceedance probabilities linearly between 0/1 # at the min/max fuzzy bounds and 0.5 at the threshold value else: truth_value = np.where( cube.data < threshold, rescale(cube.data, data_range=(bounds[0], threshold), scale_range=(0., 0.5), clip=True), rescale(cube.data, data_range=(threshold, bounds[1]), scale_range=(0.5, 1.), clip=True), ) truth_value = truth_value.astype(input_cube_dtype) # if requirement is for probabilities below threshold (rather than # above), invert the exceedance probability if self.below_thresh_ok: truth_value = 1. - truth_value cube.data = truth_value # Overwrite masked values that have been thresholded # with the un-thresholded values from the input cube. if np.ma.is_masked(cube.data): cube.data[input_cube.data.mask] = ( input_cube.data[input_cube.data.mask]) cube = self._add_threshold_coord(cube, threshold) thresholded_cubes.append(cube) cube, = thresholded_cubes.concatenate() if self.below_thresh_ok: cube.rename("probability_of_{}_below_threshold".format( cube.name())) else: cube.rename("probability_of_{}_above_threshold".format( cube.name())) cube.units = Unit(1) cube = enforce_coordinate_ordering(cube, ["realization", "percentile"]) return cube
def test_sigma(self): new_sigma = mock.Mock(units=Unit('1'), nbounds=0) self.factory.update(self.sigma, new_sigma) self.assertIs(self.factory.sigma, new_sigma)
def process( self, orography: Cube, thresholds_dict: Dict[str, List[float]], landmask: Optional[Cube] = None, ) -> Cube: """Calculate the weights depending upon where the orography point is within the topographic zones. Args: orography: Orography on standard grid. thresholds_dict: Definition of orography bands required. The expected format of the dictionary is e.g. `{'bounds': [[0, 50], [50, 200]], 'units': 'm'}` landmask: Land mask on standard grid, with land points set to one and sea points set to zero. If provided sea points are masked out in the output array. Returns: Cube containing the weights depending upon where the orography point is within the topographic zones. """ # Check that orography is a 2d cube. if len(orography.shape) != 2: msg = ("The input orography cube should be two-dimensional." "The input orography cube has {} dimensions".format( len(orography.shape))) raise InvalidCubeError(msg) # Find bands and midpoints from bounds. bands = np.array(thresholds_dict["bounds"], dtype=np.float32) threshold_units = thresholds_dict["units"] # Create topographic_zone_cube first, so that a cube is created for # each band. This will allow the data for neighbouring bands to be # put into the cube. mask_data = np.zeros(orography.shape, dtype=np.float32) topographic_zone_cubes = iris.cube.CubeList([]) for band in bands: sea_points_included = not landmask topographic_zone_cube = _make_mask_cube( mask_data, orography.coords(), band, threshold_units, sea_points_included=sea_points_included, ) topographic_zone_cubes.append(topographic_zone_cube) topographic_zone_weights = topographic_zone_cubes.concatenate_cube() topographic_zone_weights.data = topographic_zone_weights.data.astype( np.float32) # Ensure topographic_zone coordinate units is equal to orography units. topographic_zone_weights.coord("topographic_zone").convert_units( orography.units) # Read bands from cube, now that they can be guaranteed to be in the # same units as the orography. The bands are converted to a list, so # that they can be iterated through. bands = list(topographic_zone_weights.coord("topographic_zone").bounds) midpoints = topographic_zone_weights.coord("topographic_zone").points # Raise a warning, if orography extremes are outside the extremes of # the bands. if np.max(orography.data) > np.max(bands): msg = ("The maximum orography is greater than the uppermost band. " "This will potentially cause the topographic zone weights " "to not sum to 1 for a given grid point.") warnings.warn(msg) if np.min(orography.data) < np.min(bands): msg = ("The minimum orography is lower than the lowest band. " "This will potentially cause the topographic zone weights " "to not sum to 1 for a given grid point.") warnings.warn(msg) # Insert the appropriate weights into the topographic zone cube. This # includes the weights from the band that a point is in, as well as # the contribution from an adjacent band. for band_number, band in enumerate(bands): # Determine the points that are within the specified band. mask_y, mask_x = np.where((orography.data > band[0]) & (orography.data <= band[1])) orography_band = np.full(orography.shape, np.nan, dtype=np.float32) orography_band[mask_y, mask_x] = orography.data[mask_y, mask_x] # Calculate the weights. This involves calculating the # weights for all the orography but only inserting weights # that are within the band into the topographic_zone_weights cube. weights = self.calculate_weights(orography_band, band) topographic_zone_weights.data[band_number, mask_y, mask_x] = weights[mask_y, mask_x] # Calculate the contribution to the weights from the adjacent # lower band. topographic_zone_weights.data = self.add_weight_to_lower_adjacent_band( topographic_zone_weights.data, orography_band, midpoints[band_number], band_number, ) # Calculate the contribution to the weights from the adjacent # upper band. topographic_zone_weights.data = self.add_weight_to_upper_adjacent_band( topographic_zone_weights.data, orography_band, midpoints[band_number], band_number, len(bands) - 1, ) # Metadata updates topographic_zone_weights.rename("topographic_zone_weights") topographic_zone_weights.units = Unit("1") # Mask output weights using a land-sea mask. topographic_zone_masked_weights = iris.cube.CubeList([]) for topographic_zone_slice in topographic_zone_weights.slices_over( "topographic_zone"): if landmask: topographic_zone_slice.data = GenerateOrographyBandAncils( ).sea_mask(landmask.data, topographic_zone_slice.data) topographic_zone_masked_weights.append(topographic_zone_slice) topographic_zone_weights = topographic_zone_masked_weights.merge_cube() return topographic_zone_weights
def test_sigma_too_many_bounds(self): new_sigma = mock.Mock(units=Unit('1'), nbounds=4) with self.assertRaises(ValueError): self.factory.update(self.sigma, new_sigma)
def test_sigma_incompatible_units(self): new_sigma = mock.Mock(units=Unit('Pa'), nbounds=0) with self.assertRaises(ValueError): self.factory.update(self.sigma, new_sigma)
def test_eta(self): new_eta = mock.Mock(units=Unit('m'), nbounds=0) self.factory.update(self.eta, new_eta) self.assertIs(self.factory.eta, new_eta)
def units(self, units): new_units = units if isinstance(units, Unit) else Unit(units) if not new_units.is_no_unit(): raise UnitsError('CharArrays cannot have units.') self._optinfo['units'] = new_units
def test_depth(self): new_depth = mock.Mock(units=Unit('m'), nbounds=0) self.factory.update(self.depth, new_depth) self.assertIs(self.factory.depth, new_depth)
def test_non_gregorian_calendar_conversion_dtype(self): data = np.arange(4, dtype=np.float32) u1 = Unit('hours since 2000-01-01 00:00:00', calendar='360_day') u2 = Unit('hours since 2000-01-02 00:00:00', calendar='360_day') result = u1.convert(data, u2) self.assertEqual(result.dtype, np.float32)
q = iris.load('p_lvl_data.nc', 'q')[0][step] # Specific humidity # -------------------------------------------------------------------- # --------------------- Define Other Variables ----------------------- pressure_levels = [300, 350, 400, 450, 500, 550, 600, 650, 700, 750, 800, 850, 900, 925, 950, 975, 1000] g = 9.807 # Gravity e0 = 6.11 # Saturation vapour pressure at T0 in hPa eps = 0.622 # Ratio of Rd/Rv in kg/kg # -------------------------------------------------------------------- # ------------------- Calculate Derived Variables -------------------- # Create cube with all entries zero s_zero = (t2m.copy() * 0) # Same dimensions as single level data p_zero = (t.copy() * 0) # Same dimensions as pressure level data s_zero.units = Unit('Unknown') # Erase units p_zero.units = Unit('Unknown') # Erase units # Calculate variables e_sfc = s_zero.copy() q_sfc = s_zero.copy() e_sfc.data = e0 * np.exp((17.27 * d2m.data)/(d2m.data + 237.3)) q_sfc.data = (eps * e_sfc.data)/(sp.data - ((1-eps) * e_sfc.data)) # -------------------------------------------------------------------- p = p_zero for i in range(17): p.data[i] = pressure_levels[i] # ------------------------ Calculate Variable ------------------------ # Calculate integrand for max saturation on pressure levels integrand_p = p_zero.copy()
def _check_variable_attrs(dataset, var_name, required_attributes=None): ''' Convenience method to check a variable attributes based on the expected_vars dict ''' score = 0 out_of = 0 messages = [] if var_name not in dataset.variables: # No need to check the attrs if the variable doesn't exist return (score, out_of, messages) var = dataset.variables[var_name] # Get the expected attrs to check check_attrs = required_attributes or required_var_attrs.get(var_name, {}) var_attrs = set(var.ncattrs()) for attr in check_attrs: if attr == 'dtype': # dtype check is special, see above continue out_of += 1 score += 1 # Check if the attribute is present if attr not in var_attrs: messages.append('Variable {} must contain attribute: {}' ''.format(var_name, attr)) score -= 1 continue # Attribute exists, let's check if there was a value we need to compare against if check_attrs[attr] is not None: if getattr(var, attr) != check_attrs[attr]: # No match, this may be an error, but first an exception for units if attr == 'units': msg = ('Variable {} units attribute must be ' 'convertible to {}'.format(var_name, check_attrs[attr])) try: cur_unit = Unit(var.units) comp_unit = Unit(check_attrs[attr]) if not cur_unit.is_convertible(comp_unit): messages.append(msg) score -= 1 except ValueError: messages.append(msg) score -= 1 else: messages.append('Variable {} attribute {} must be {}'.format(var_name, attr, check_attrs[attr])) score -= 1 else: # Final check to make sure the attribute isn't an empty string try: # try stripping whitespace, and return an error if empty att_strip = getattr(var, attr).strip() if not att_strip: messages.append('Variable {} attribute {} is empty' ''.format(var_name, attr)) score -= 1 except AttributeError: pass return (score, out_of, messages)
def test_depth_incompatible_units(self): self.depth.units = Unit('km') with self.assertRaises(ValueError): OceanSigmaFactory(**self.kwargs)
class TestCategoriseCoordFunctionForTime: def __init__(self): self.u = Unit('days since 1600-01-01 00:00:00', calendar=CALENDAR_STANDARD) self.points = np.arange(1, 5, 1) self.coord = iris.coords.DimCoord(self.points, units=self.u) self.start = datetime.datetime(2000, 1, 1) self.end = datetime.datetime(2003, 4, 24) self.start = self.u.date2num(self.start) self.end = self.u.date2num(self.end) def setup_func(self): self.__init__() @with_setup(setup_func) def test_categorise_coord_function_time_year_only(self): delta = date_delta_creator(1) result_function = categorise_coord_function(self.start, self.end, delta, True) expected = np.array([self.u.date2num(datetime.datetime(2000, 7, 1, 0, 0, 0)), self.u.date2num(datetime.datetime(2001, 7, 1, 0, 0, 0)), self.u.date2num(datetime.datetime(2002, 7, 1, 0, 0, 0))]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2000, 1, 1, 0, 0, 0))), expected[0]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2001, 3, 3, 0, 0, 0))), expected[1]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2002, 5, 8, 0, 0, 0))), expected[2]) @with_setup(setup_func) def test_categorise_coord_function_time_year_month(self): delta = date_delta_creator(1, 1) result_function = categorise_coord_function(self.start, self.end, delta, True) expected = np.array([self.u.date2num(datetime.datetime(2000, 7, 15, 0, 0, 0)), self.u.date2num(datetime.datetime(2001, 8, 15, 0, 0, 0)), self.u.date2num(datetime.datetime(2002, 9, 15, 0, 0, 0))]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2000, 1, 1, 0, 0, 0))), expected[0]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2001, 3, 3, 0, 0, 0))), expected[1]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2002, 5, 8, 0, 0, 0))), expected[2]) @with_setup(setup_func) def test_categorise_coord_function_with_month_going_past_december(self): start = datetime.datetime(2000, 11, 3) start = self.u.date2num(start) end = datetime.datetime(2004, 11, 3) end = self.u.date2num(end) delta = date_delta_creator(1, 2) result_function = categorise_coord_function(start, end, delta, True) expected = np.array([self.u.date2num(datetime.datetime(2001, 6, 3, 0, 0, 0)), self.u.date2num(datetime.datetime(2002, 8, 3, 0, 0, 0)), self.u.date2num(datetime.datetime(2003, 10, 3, 0, 0, 0))]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(1999, 1, 1, 0, 0, 0))), expected[0]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2001, 3, 3, 0, 0, 0))), expected[0]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2002, 3, 3, 0, 0, 0))), expected[1]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2005, 5, 8, 0, 0, 0))), expected[2]) @with_setup(setup_func) def test_categorise_coord_function_time_year_month_day_hour_minute_second(self): delta = date_delta_creator(1, 3, 2, 4, 5, 6) result_function = categorise_coord_function(self.start, self.end, delta, True) expected = np.array([self.u.date2num(datetime.datetime(2000, 8, 16, 2, 2, 33)), self.u.date2num(datetime.datetime(2001, 11, 18, 6, 7, 39)), self.u.date2num(datetime.datetime(2003, 2, 20, 10, 12, 45))]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2000, 1, 1, 0, 0, 0))), expected[0]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2001, 7, 3, 0, 0, 0))), expected[1]) assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2002, 9, 8, 0, 0, 0))), expected[2])
def setUp(self): self.sigma = mock.Mock(units=Unit('1'), nbounds=0) self.eta = mock.Mock(units=Unit('m'), nbounds=0) self.depth = mock.Mock(units=Unit('m'), nbounds=0) self.kwargs = dict(sigma=self.sigma, eta=self.eta, depth=self.depth)
def test_fail(self): units = Unit('days since 1970-01-02 00:00:00', calendar='gregorian') cube2 = Cube([1, 2, 3], 'air_temperature', units='K') cube2.add_dim_coord(DimCoord([0, 1, 2], 'time', units=units), 0) with self.assertRaises(iris.exceptions.ConcatenateError): CubeList([self.cube1, cube2]).concatenate_cube()
def set_up_cube(num_time_points=1, num_grid_points=1, num_height_levels=7, data=None, name=None, unit=None, height=None): """Set up a UK model standard grid (UKVX) cube.""" cubel = iris.cube.CubeList() tunit = Unit("hours since 1970-01-01 00:00:00", "gregorian") t_0 = 402192.5 if isinstance(num_grid_points, int): num_grid_points_x = num_grid_points_y = num_grid_points else: num_grid_points_x = num_grid_points[0] num_grid_points_y = num_grid_points[1] for i_idx in range(num_height_levels): cubel1 = iris.cube.CubeList() for j_idx in range(num_time_points): cube = _make_ukvx_grid() cube = cube[:num_grid_points_x, :num_grid_points_y] cube.add_aux_coord(AuxCoord(t_0 + j_idx, "time", units=tunit)) if height is None: cube.add_aux_coord(AuxCoord(i_idx, "model_level_number")) elif isinstance(height, float) or isinstance(height, int): cube.add_aux_coord( AuxCoord(height, "height", units=Unit("meter"))) else: cube.add_aux_coord( AuxCoord(height[i_idx], "height", units=Unit("meter"))) cubel1.append(cube) cubel.append(cubel1.merge_cube()) cubel = cubel.merge(0) cube = cubel[0] if data is not None: try: data = np.array(data, dtype=np.float32) cube.data = data.reshape(cube.data.shape) except ValueError as err: if err == "total size of new array must be unchanged": msg = ("supplied data does not fit the cube." "cube dimensions: {} vs. supplied data {}") raise ValueError(msg.format(cube.shape, data.shape)) raise ValueError(err) if name is not None: try: cube.standard_name = name except ValueError as err: msg = ("error trying to set the supplied name as cube data name: " "{}".format(err)) raise ValueError(msg) except TypeError as err: msg = ("error trying to set the supplied name as cube data name: " "the name should be string and have a valid variable name " "{}".format(err)) raise ValueError(msg) if unit is not None: try: cube.units = Unit(unit) except ValueError as err: msg = "error trying to set Units to cube. supplied unit: {}" raise ValueError(msg.format(unit)) return cube
def setUp(self): self.units = Unit('days since 1970-01-01 00:00:00', calendar='gregorian') self.cube1 = Cube([1, 2, 3], 'air_temperature', units='K') self.cube1.add_dim_coord(DimCoord([0, 1, 2], 'time', units=self.units), 0)
def process(self, input_cube): """Convert each point to a truth value based on provided threshold values. The truth value may or may not be fuzzy depending upon if fuzzy_bounds are supplied. If the plugin has a "threshold_units" member, this is used to convert both thresholds and fuzzy bounds into the units of the input cube. Args: input_cube (iris.cube.Cube): Cube to threshold. The code is dimension-agnostic. Returns: cube (iris.cube.Cube): Cube after a threshold has been applied. The data within this cube will contain values between 0 and 1 to indicate whether a given threshold has been exceeded or not. The cube meta-data will contain: * Input_cube name prepended with probability_of_X_above(or below)_threshold (where X is the diagnostic under consideration) * Threshold dimension coordinate with same units as input_cube * Threshold attribute (above or below threshold) * Cube units set to (1). Raises: ValueError: if a np.nan value is detected within the input cube. """ # Record input cube data type to ensure consistent output, though # integer data must become float to enable fuzzy thresholding. input_cube_dtype = input_cube.dtype if input_cube.dtype.kind == 'i': input_cube_dtype = np.float32 thresholded_cubes = iris.cube.CubeList() if np.isnan(input_cube.data).any(): raise ValueError("Error: NaN detected in input cube data") # if necessary, convert thresholds and fuzzy bounds into cube units if self.threshold_units is not None: self.thresholds = [ self.threshold_units.convert(threshold, input_cube.units) for threshold in self.thresholds ] self.fuzzy_bounds = [ tuple([ self.threshold_units.convert(threshold, input_cube.units) for threshold in bounds ]) for bounds in self.fuzzy_bounds ] # set name of threshold coordinate to match input diagnostic self.threshold_coord_name = input_cube.name() # apply fuzzy thresholding for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds): cube = input_cube.copy() # if upper and lower bounds are equal, set a deterministic 0/1 # probability based on exceedance of the threshold if bounds[0] == bounds[1]: truth_value = cube.data > threshold # otherwise, scale exceedance probabilities linearly between 0/1 # at the min/max fuzzy bounds and 0.5 at the threshold value else: truth_value = np.where( cube.data < threshold, rescale(cube.data, data_range=(bounds[0], threshold), scale_range=(0., 0.5), clip=True), rescale(cube.data, data_range=(threshold, bounds[1]), scale_range=(0.5, 1.), clip=True), ) truth_value = truth_value.astype(input_cube_dtype) # if requirement is for probabilities below threshold (rather than # above), invert the exceedance probability if self.below_thresh_ok: truth_value = 1. - truth_value cube.data = truth_value # Overwrite masked values that have been thresholded # with the un-thresholded values from the input cube. if np.ma.is_masked(cube.data): cube.data[input_cube.data.mask] = ( input_cube.data[input_cube.data.mask]) cube = self._add_threshold_coord(cube, threshold) thresholded_cubes.append(cube) cube, = thresholded_cubes.concatenate() if self.below_thresh_ok: cube.rename("probability_of_{}_below_threshold".format( cube.name())) else: cube.rename("probability_of_{}_above_threshold".format( cube.name())) cube.units = Unit(1) cube = enforce_coordinate_ordering(cube, ["realization", "percentile"]) return cube
def test_depth_incompatible_units(self): new_depth = mock.Mock(units=Unit('Pa'), nbounds=0) with self.assertRaises(ValueError): self.factory.update(self.depth, new_depth)
def test_basic(self): """Test that the result is a numpy array.""" cube_name = "air_temperature" cube_units = Unit("degreesC") result = get_bounds_of_distribution(cube_name, cube_units) self.assertIsInstance(result, np.ndarray)