예제 #1
0
 def test_kwarg_pass_through_no_kwargs(self):
     call_func = mock.Mock()
     data = mock.sentinel.data
     axis = mock.sentinel.axis
     aggregator = Aggregator('', call_func)
     aggregator.aggregate(data, axis)
     call_func.assert_called_once_with(data, axis=axis)
예제 #2
0
 def test_no_units_change(self):
     # If the Aggregator has no units_func then the units should be
     # left unchanged.
     aggregator = Aggregator("", None)
     cube = mock.Mock(units=mock.sentinel.units)
     aggregator.update_metadata(cube, [])
     self.assertIs(cube.units, mock.sentinel.units)
예제 #3
0
 def test_mdtol_intercept(self):
     call_func = mock.Mock()
     data = mock.sentinel.data
     axis = mock.sentinel.axis
     aggregator = Aggregator("", call_func)
     aggregator.aggregate(data, axis, wibble="wobble", mdtol=0.8)
     call_func.assert_called_once_with(data, axis=axis, wibble="wobble")
예제 #4
0
 def test_mdtol_intercept(self):
     call_func = Mock()
     data = sentinel.data
     axis = sentinel.axis
     aggregator = Aggregator('', call_func)
     aggregator.aggregate(data, axis, wibble='wobble', mdtol=0.8)
     call_func.assert_called_once_with(data, axis=axis, wibble='wobble')
예제 #5
0
 def test_kwarg_pass_through_no_kwargs(self):
     call_func = Mock()
     data = sentinel.data
     axis = sentinel.axis
     aggregator = Aggregator('', call_func)
     aggregator.aggregate(data, axis)
     call_func.assert_called_once_with(data, axis=axis)
예제 #6
0
 def test_mdtol_intercept(self):
     call_func = Mock()
     data = sentinel.data
     axis = sentinel.axis
     aggregator = Aggregator('', call_func)
     aggregator.aggregate(data, axis, wibble='wobble', mdtol=0.8)
     call_func.assert_called_once_with(data, axis=axis, wibble='wobble')
예제 #7
0
 def test_no_units_change(self):
     # If the Aggregator has no units_func then the units should be
     # left unchanged.
     aggregator = Aggregator('', None)
     cube = Mock(units=sentinel.units)
     aggregator.update_metadata(cube, [])
     self.assertIs(cube.units, sentinel.units)
예제 #8
0
 def test_kwarg_pass_through_no_kwargs(self):
     lazy_func = Mock()
     data = sentinel.data
     axis = sentinel.axis
     aggregator = Aggregator('', None, lazy_func=lazy_func)
     aggregator.lazy_aggregate(data, axis)
     lazy_func.assert_called_once_with(data, axis)
예제 #9
0
 def test_kwarg_pass_through_no_kwargs(self):
     lazy_func = mock.Mock()
     data = mock.sentinel.data
     axis = mock.sentinel.axis
     aggregator = Aggregator("", None, lazy_func=lazy_func)
     aggregator.lazy_aggregate(data, axis)
     lazy_func.assert_called_once_with(data, axis=axis)
예제 #10
0
 def test_kwarg_pass_through_init_kwargs(self):
     call_func = mock.Mock()
     data = mock.sentinel.data
     axis = mock.sentinel.axis
     kwargs = dict(wibble='wobble', foo='bar')
     aggregator = Aggregator('', call_func, **kwargs)
     aggregator.aggregate(data, axis)
     call_func.assert_called_once_with(data, axis=axis, **kwargs)
예제 #11
0
 def setUp(self):
     self.TEST = Aggregator('test', None)
     self.array = ma.array([[1, 2, 3], [4, 5, 6]],
                           mask=[[False, True, False], [True, False,
                                                        False]],
                           dtype=np.float64)
     self.expected_result_axis0 = ma.array([1, 2, 3], mask=None)
     self.expected_result_axis1 = ma.array([4, 5], mask=None)
예제 #12
0
 def test_kwarg_pass_through_call_kwargs(self):
     call_func = mock.Mock()
     data = mock.sentinel.data
     axis = mock.sentinel.axis
     kwargs = dict(wibble="wobble", foo="bar")
     aggregator = Aggregator("", call_func)
     aggregator.aggregate(data, axis, **kwargs)
     call_func.assert_called_once_with(data, axis=axis, **kwargs)
예제 #13
0
 def test_kwarg_pass_through_call_kwargs(self):
     call_func = Mock()
     data = sentinel.data
     axis = sentinel.axis
     kwargs = dict(wibble='wobble', foo='bar')
     aggregator = Aggregator('', call_func)
     aggregator.aggregate(data, axis, **kwargs)
     call_func.assert_called_once_with(data, axis=axis, **kwargs)
예제 #14
0
 def test_kwarg_pass_through_call_kwargs(self):
     lazy_func = mock.Mock()
     data = mock.sentinel.data
     axis = mock.sentinel.axis
     kwargs = dict(wibble='wobble', foo='bar')
     aggregator = Aggregator('', None, lazy_func=lazy_func)
     aggregator.lazy_aggregate(data, axis, **kwargs)
     lazy_func.assert_called_once_with(data, axis=axis, **kwargs)
예제 #15
0
 def test_kwarg_pass_through_init_kwargs(self):
     lazy_func = Mock()
     data = sentinel.data
     axis = sentinel.axis
     kwargs = dict(wibble='wobble', foo='bar')
     aggregator = Aggregator('', None, lazy_func=lazy_func, **kwargs)
     aggregator.lazy_aggregate(data, axis)
     lazy_func.assert_called_once_with(data, axis, **kwargs)
예제 #16
0
 def test_kwarg_pass_through_init_kwargs(self):
     lazy_func = mock.Mock()
     data = mock.sentinel.data
     axis = mock.sentinel.axis
     kwargs = dict(wibble="wobble", foo="bar")
     aggregator = Aggregator("", None, lazy_func=lazy_func, **kwargs)
     aggregator.lazy_aggregate(data, axis)
     lazy_func.assert_called_once_with(data, axis=axis, **kwargs)
예제 #17
0
 def test_units_change(self):
     # If the Aggregator has a units_func then the new units should
     # be defined by its return value.
     units_func = mock.Mock(return_value=mock.sentinel.new_units)
     aggregator = Aggregator("", None, units_func)
     cube = mock.Mock(units=mock.sentinel.units)
     aggregator.update_metadata(cube, [])
     units_func.assert_called_once_with(mock.sentinel.units)
     self.assertEqual(cube.units, mock.sentinel.new_units)
예제 #18
0
 def test_units_change(self):
     # If the Aggregator has a units_func then the new units should
     # be defined by its return value.
     units_func = Mock(return_value=sentinel.new_units)
     aggregator = Aggregator('', None, units_func)
     cube = Mock(units=sentinel.units)
     aggregator.update_metadata(cube, [])
     units_func.assert_called_once_with(sentinel.units)
     self.assertEqual(cube.units, sentinel.new_units)
예제 #19
0
 def test_kwarg_pass_through_combined_kwargs(self):
     call_func = mock.Mock()
     data = mock.sentinel.data
     axis = mock.sentinel.axis
     init_kwargs = dict(wibble='wobble', var=1.0)
     call_kwargs = dict(foo='foo', var=0.5)
     aggregator = Aggregator('', call_func, **init_kwargs)
     aggregator.aggregate(data, axis, **call_kwargs)
     expected_kwargs = init_kwargs.copy()
     expected_kwargs.update(call_kwargs)
     call_func.assert_called_once_with(data, axis=axis, **expected_kwargs)
예제 #20
0
 def test_kwarg_pass_through_combined_kwargs(self):
     lazy_func = Mock()
     data = sentinel.data
     axis = sentinel.axis
     init_kwargs = dict(wibble='wobble', var=1.0)
     call_kwargs = dict(foo='foo', var=0.5)
     aggregator = Aggregator('', None, lazy_func=lazy_func, **init_kwargs)
     aggregator.lazy_aggregate(data, axis, **call_kwargs)
     expected_kwargs = init_kwargs.copy()
     expected_kwargs.update(call_kwargs)
     lazy_func.assert_called_once_with(data, axis, **expected_kwargs)
예제 #21
0
 def test_kwarg_pass_through_combined_kwargs(self):
     lazy_func = mock.Mock()
     data = mock.sentinel.data
     axis = mock.sentinel.axis
     init_kwargs = dict(wibble="wobble", var=1.0)
     call_kwargs = dict(foo="foo", var=0.5)
     aggregator = Aggregator("", None, lazy_func=lazy_func, **init_kwargs)
     aggregator.lazy_aggregate(data, axis, **call_kwargs)
     expected_kwargs = init_kwargs.copy()
     expected_kwargs.update(call_kwargs)
     lazy_func.assert_called_once_with(data, axis=axis, **expected_kwargs)
예제 #22
0
    def percentile_weighted_mean(self, cube, weights):
        """
        Blend percentile data using the weights provided.

        Args:
            cube (iris.cube.Cube):
                The cube which is being blended over self.blend_coord. Assumes
                self.blend_coord and percentile are leading coordinates (enforced
                in process).
            weights (iris.cube.Cube):
                Cube of blending weights.
        Returns:
            iris.cube.Cube:
                The cube with percentile values blended over self.blend_coord,
                with suitable weightings applied.
        """
        non_perc_slice = next(cube.slices_over(PERC_COORD))
        weights_array = self.get_weights_array(non_perc_slice, weights)
        weights_array = self._normalise_weights(weights_array)

        # Set up aggregator
        PERCENTILE_BLEND = Aggregator(
            "mean",  # Use CF-compliant cell method.
            PercentileBlendingAggregator.aggregate,
        )

        cube_new = collapsed(
            cube,
            self.blend_coord,
            PERCENTILE_BLEND,
            percentiles=cube.coord(PERC_COORD).points,
            arr_weights=weights_array,
        )

        return cube_new
예제 #23
0
    def weighted_maximum(self, cube, weights):
        """
        Blend data using a weighted maximum using the weights provided.
        This entails scaling the data by the weights before then taking
        a maximum across the blending coordinate self.coord.

        Args:
            cube (iris.cube.Cube):
                The cube which is being blended over self.coord.
            weights (iris.cube.Cube):
                Cube of blending weights.
        Returns:
            cube_new (iris.cube.Cube):
                The cube with values blended over self.coord, with suitable
                weightings applied.
        """

        weights_array = self.non_percentile_weights(cube,
                                                    weights,
                                                    custom_aggregator=True)
        # Set up aggregator
        MAX_PROBABILITY = (
            Aggregator(
                'maximum',  # Use CF-compliant cell method.
                MaxProbabilityAggregator.aggregate))

        cube_new = cube.collapsed(self.coord,
                                  MAX_PROBABILITY,
                                  arr_weights=weights_array)
        cube_new.data = cube_new.data.astype(np.float32)
        return cube_new
예제 #24
0
def main():
    # Load the whole time-sequence as a single cube.
    file_path = iris.sample_data_path("E1_north_america.nc")
    cube = iris.load_cube(file_path)

    # Make an aggregator from the user function.
    SPELL_COUNT = Aggregator(
        "spell_count", count_spells, units_func=lambda units: 1
    )

    # Define the parameters of the test.
    threshold_temperature = 280.0
    spell_years = 5

    # Calculate the statistic.
    warm_periods = cube.collapsed(
        "time",
        SPELL_COUNT,
        threshold=threshold_temperature,
        spell_length=spell_years,
    )
    warm_periods.rename("Number of 5-year warm spells in 240 years")

    # Plot the results.
    qplt.contourf(warm_periods, cmap="RdYlBu_r")
    plt.gca().coastlines()
    iplt.show()
예제 #25
0
def window_counts(mycube, value_threshold, window_size, pctile):
    """
    Function that returns a flat array containing
    the number of data points within a time window `window_size'
    per grid point that satify a condition
    value > value_threshold.
    It also returns statistical measures for the flat array
    window_counts[0] = array
    window_counts[1] = mean(array)
    window_counts[2] = std(array)
    window_counts[3] = percentile(array, pctile)
    """

    # Make an aggregator from the user function.
    SPELL_COUNT = Aggregator('spell_count',
                             count_spells,
                             units_func=lambda units: 1)

    # Calculate the statistic.
    counts_windowed_cube = mycube.collapsed('time',
                                            SPELL_COUNT,
                                            threshold=value_threshold,
                                            spell_length=window_size)

    #if one wants to print the whole array
    #np.set_printoptions(threshold=np.nan)
    r = counts_windowed_cube.data.flatten()
    meanr = np.mean(r)
    stdr = np.std(r)
    prcr = np.percentile(r, pctile)
    return r, meanr, stdr, prcr
예제 #26
0
    def __init__(self):
        """Create an aggregator instance for reuse"""
        self.aggregator_instance = Aggregator("mode", self.mode_aggregator)

        # Create the expected cell method for use with single cube inputs
        # that do not pass through the aggregator.
        self.mode_cell_method = iris.coords.CellMethod("mode", coords="time")
예제 #27
0
 def test_fail_no_lazy(self):
     dummy_agg = Aggregator('custom_op', lambda x: 1)
     with self.assertRaises(LazyAggregatorError) as err:
         cube_collapsed = self.cube.collapsed('x', dummy_agg, lazy=True)
     msg = err.exception.message
     self.assertIn('custom_op', msg)
     self.assertIn('lazy', msg)
     self.assertIn('not support', msg)
예제 #28
0
 def setUp(self):
     self.TEST = Aggregator('test', None)
     self.array = ma.array([[1, 2, 3],
                            [4, 5, 6]],
                           mask=[[False, True, False],
                                 [True, False, False]],
                           dtype=np.float64)
     self.expected_result_axis0 = ma.array([1, 2, 3], mask=None)
     self.expected_result_axis1 = ma.array([4, 5], mask=None)
예제 #29
0
    def percentile_weighted_mean(self, cube, weights, perc_coord):
        """
        Blend percentile data using the weights provided.

        Args:
            cube (iris.cube.Cube):
                The cube which is being blended over self.blend_coord.
            weights (iris.cube.Cube):
                Cube of blending weights.
            perc_coord (iris.coords.DimCoord):
                The percentile coordinate for this cube.
        Returns:
            iris.cube.Cube:
                The cube with percentile values blended over self.blend_coord,
                with suitable weightings applied.
        """
        percentiles = np.array(perc_coord.points, dtype=np.float32)
        (perc_dim, ) = cube.coord_dims(perc_coord.name())

        # The iris.analysis.Aggregator moves the coordinate being
        # collapsed to index=-1 in initialisation, before the
        # aggregation method is called. This reduces by 1 the index
        # of all coordinates with an initial index higher than the
        # collapsing coordinate. As we need to know the index of
        # the percentile coordinate at a later step, if it will be
        # changed by this process, we adjust our record (perc_dim)
        # here.
        if cube.coord_dims(self.blend_coord)[0] < perc_dim:
            perc_dim -= 1

        weights_array = self.percentile_weights(cube, weights, perc_coord)

        # Set up aggregator
        PERCENTILE_BLEND = Aggregator(
            "mean",  # Use CF-compliant cell method.
            PercentileBlendingAggregator.aggregate,
        )

        cube_new = collapsed(
            cube,
            self.blend_coord,
            PERCENTILE_BLEND,
            arr_percent=percentiles,
            arr_weights=weights_array,
            perc_dim=perc_dim,
        )

        cube_new.data = cube_new.data.astype(np.float32)
        # Ensure collapsed coordinates do not promote themselves
        # to float64.
        for coord in cube_new.coords():
            if coord.points.dtype == np.float64:
                coord.points = coord.points.astype(np.float32)
        return cube_new
예제 #30
0
def mask_cube_counts(mycube, value_threshold, counts_threshold, window_size):
    # Make an aggregator from the user function.
    SPELL_COUNT = Aggregator('spell_count',
                             count_spells,
                             units_func=lambda units: 1)

    # Calculate the statistic.
    counts_windowed_cube = mycube.collapsed('time',
                                            SPELL_COUNT,
                                            threshold=value_threshold,
                                            spell_length=window_size)

    mask = counts_windowed_cube.data > counts_threshold
    mask.astype(np.int)
    # preserving the original cube metadata
    masked_cube = mycube.copy()
    masked_cube.data = mycube.data * mask
    return counts_windowed_cube, mask, masked_cube
예제 #31
0
def _get_drought_data(cfg, cube):
    """Prepare data and calculate characteristics."""
    # make a new cube to increase the size of the data array
    # Make an aggregator from the user function.
    spell_no = Aggregator('spell_count',
                          count_spells,
                          units_func=lambda units: 1)
    new_cube = _make_new_cube(cube)

    # calculate the number of drought events and their average duration
    drought_show = new_cube.collapsed('time', spell_no,
                                      threshold=cfg['threshold'])
    drought_show.rename('Drought characteristics')
    # length of time series
    time_length = len(new_cube.coord('time').points) / 12.0
    # Convert number of droughtevents to frequency (per year)
    drought_show.data[:, :, 0] = drought_show.data[:, :,
                                                   0] / time_length
    return drought_show
예제 #32
0
def _get_fillvalues_mask(cube, threshold_fraction, min_value, time_window):
    """
    Compute the per-model missing values mask.

    Construct the mask that fills a certain time window with missing values
    if the number of values in that specific window is less than a given
    fractional threshold; it uses a custom iris Aggregator function that
    aggregates the cube data by a given time window and counts the number of
    valid (unmasked) data points within that window;
    a simple value thresholding is also applied if needed.
    """
    # basic checks
    if threshold_fraction < 0 or threshold_fraction > 1.0:
        raise ValueError(
            "Fraction of missing values {} should be between 0 and 1.0".format(
                threshold_fraction))
    nr_time_points = len(cube.coord('time').points)
    if time_window > nr_time_points:
        msg = "Time window (in time units) larger than total time span. Stop."
        raise ValueError(msg)

    max_counts_per_time_window = nr_time_points / time_window
    # round to lower integer
    counts_threshold = int(max_counts_per_time_window * threshold_fraction)

    # Make an aggregator
    spell_count = Aggregator('spell_count',
                             count_spells,
                             units_func=lambda units: 1)

    # Calculate the statistic.
    counts_windowed_cube = cube.collapsed('time',
                                          spell_count,
                                          threshold=min_value,
                                          spell_length=time_window)

    # Create mask
    mask = counts_windowed_cube.data < counts_threshold
    if np.ma.isMaskedArray(mask):
        mask = mask.data | mask.mask

    return mask
예제 #33
0
def main():

    file_path = iris.sample_data_path(
        '/nfs/a266/data/CMIP5_AFRICA/BC_0.5x0.5/IPSL-CM5A-LR/historical/pr_WFDEI_1979-2013_0.5x0.5_day_IPSL-CM5A-LR_africa_historical_r1i1p1_full.nc'
    )

    cube = iris.load_cube(file_path)

    cube_wafr = cube.intersection(latitude=(-10.0, 10.0),
                                  longitude=(4.0, 25.0))

    iris.coord_categorisation.add_year(cube_wafr, 'time', name='year')

    iris.coord_categorisation.add_month_number(cube_wafr,
                                               'time',
                                               name='month_number')

    iris.coord_categorisation.add_season(cube_wafr, 'time', name='season')

    SPELL_COUNT = Aggregator('spell_count',
                             count_spells,
                             units_func=lambda units: 1)

    threshold_rainfall = 0.1

    spell_days = 10

    dry_periods = cube.collapsed('time',
                                 SPELL_COUNT,
                                 threshold=threshold_rainfall,
                                 spell_length=spell_days)

    dry_periods.rename('Number of 10-days dry spells in 35 years')

    qplt.contourf(dry_periods, cmap='RdYlBu_r')

    plt.gca().coastlines()

    iplt.show()
예제 #34
0
    def __init__(self,
                 model_id_attr: Optional[str] = None,
                 record_run_attr: Optional[str] = None):
        """
        Set up plugin and create an aggregator instance for reuse

        Args:
            model_id_attr:
                Name of attribute recording source models that should be
                inherited by the output cube. The source models are expected as
                a space-separated string.
            record_run_attr:
                Name of attribute used to record models and cycles used in
                constructing the weather symbols.
        """
        self.aggregator_instance = Aggregator("mode", self.mode_aggregator)

        self.model_id_attr = model_id_attr
        self.record_run_attr = record_run_attr

        # Create the expected cell method for use with single cube inputs
        # that do not pass through the aggregator.
        self.mode_cell_method = iris.coords.CellMethod("mode", coords="time")
예제 #35
0
class Test_aggregate(tests.IrisTest):
    # These unit tests don't call a data aggregation function, they call a
    # mocked one i.e. the return values of the mocked data aggregation
    # function don't matter, only how these are dealt with by the aggregate
    # method.
    def setUp(self):
        self.TEST = Aggregator('test', None)
        self.array = ma.array([[1, 2, 3], [4, 5, 6]],
                              mask=[[False, True, False], [True, False,
                                                           False]],
                              dtype=np.float64)
        self.expected_result_axis0 = ma.array([1, 2, 3], mask=None)
        self.expected_result_axis1 = ma.array([4, 5], mask=None)

    def test_masked_notol(self):
        # Providing masked array with no tolerance keyword (mdtol) provided.
        axis = 0
        mock_return = self.expected_result_axis0.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis)
            self.assertMaskedArrayEqual(result, self.expected_result_axis0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_above_tol(self):
        # Providing masked array with a high tolerance (mdtol) provided.
        axis = 0
        mock_return = self.expected_result_axis0.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.55)
            self.assertMaskedArrayEqual(result, self.expected_result_axis0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.55)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_below_tol(self):
        # Providing masked array with a tolerance on missing values, low
        # enough to modify the resulting mask for axis 0.
        axis = 0
        result_axis_0 = self.expected_result_axis0.copy()
        result_axis_0.mask = np.array([True, True, False])
        mock_return = ma.array([1, 2, 3], mask=None)
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayAlmostEqual(result, result_axis_0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_below_tol_alt(self):
        # Providing masked array with a tolerance on missing values, low
        # enough to modify the resulting mask for axis 1.
        axis = 1
        result_axis_1 = self.expected_result_axis1.copy()
        result_axis_1.mask = np.array([True, True])
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.1)
            self.assertMaskedArrayAlmostEqual(result, result_axis_1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_unmasked_with_mdtol(self):
        # Providing aggregator with an unmasked array and tolerance specified
        # for missing data - ensure that result is unaffected.
        data = self.array.data

        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0.5)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0.5)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

    def test_unmasked(self):
        # Providing aggregator with an unmasked array and no additional keyword
        # arguments ensure that result is unaffected.
        data = self.array.data

        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_array_len_one_mdtol(self):
        # Test the case when the data aggregation function returns a scalar and
        # turns it into a masked array.
        axis = -1
        data = self.array.flatten()
        mock_return = 2
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=1)
            self.assertMaskedArrayEqual(result, ma.array([2], mask=[False]))
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_array_len_one_mdtol_alt(self):
        # Test the case when the data aggregation function returns a scalar
        # with no tolerance for missing data values and turns it into a masked
        # array of length one.
        axis = -1
        data = self.array.flatten()
        mock_return = 2
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0)
            self.assertMaskedArrayEqual(result, ma.array([2], mask=[True]))
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_non_masked_array_from_masked_array(self):
        # Providing a masked array, call_func returning a non-masked array,
        # resulting in a masked array output.
        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        result_axis_0 = ma.array(mock_return, mask=[True, True, False])
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayAlmostEqual(result, result_axis_0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)
예제 #36
0
class Test_aggregate(tests.IrisTest):
    # These unit tests don't call a data aggregation function, they call a
    # mocked one i.e. the return values of the mocked data aggregation
    # function don't matter, only how these are dealt with by the aggregate
    # method.
    def setUp(self):
        self.TEST = Aggregator("test", None)
        self.array = ma.array(
            [[1, 2, 3], [4, 5, 6]],
            mask=[[False, True, False], [True, False, False]],
            dtype=np.float64,
        )
        self.expected_result_axis0 = ma.array([1, 2, 3], mask=None)
        self.expected_result_axis1 = ma.array([4, 5], mask=None)

    def test_masked_notol(self):
        # Providing masked array with no tolerance keyword (mdtol) provided.
        axis = 0
        mock_return = self.expected_result_axis0.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis)
            self.assertMaskedArrayEqual(result, self.expected_result_axis0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_above_tol(self):
        # Providing masked array with a high tolerance (mdtol) provided.
        axis = 0
        mock_return = self.expected_result_axis0.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=0.55)
            self.assertMaskedArrayEqual(result, self.expected_result_axis0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=0.55)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_below_tol(self):
        # Providing masked array with a tolerance on missing values, low
        # enough to modify the resulting mask for axis 0.
        axis = 0
        result_axis_0 = self.expected_result_axis0.copy()
        result_axis_0.mask = np.array([True, True, False])
        mock_return = ma.array([1, 2, 3], mask=None)
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=0.45)
            self.assertMaskedArrayAlmostEqual(result, result_axis_0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=0.45)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_below_tol_alt(self):
        # Providing masked array with a tolerance on missing values, low
        # enough to modify the resulting mask for axis 1.
        axis = 1
        result_axis_1 = self.expected_result_axis1.copy()
        result_axis_1.mask = np.array([True, True])
        mock_return = self.expected_result_axis1.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=0.1)
            self.assertMaskedArrayAlmostEqual(result, result_axis_1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_unmasked_with_mdtol(self):
        # Providing aggregator with an unmasked array and tolerance specified
        # for missing data - ensure that result is unaffected.
        data = self.array.data

        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0.5)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0.5)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

    def test_unmasked(self):
        # Providing aggregator with an unmasked array and no additional keyword
        # arguments ensure that result is unaffected.
        data = self.array.data

        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_scalar_mdtol(self):
        # Test the case when the data aggregation function returns a scalar and
        # turns it into a masked array.
        axis = -1
        data = self.array.flatten()
        mock_return = 2
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=1)
            self.assertMaskedArrayEqual(result, ma.array(2, mask=False))
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_scalar_mdtol_alt(self):
        # Test the case when the data aggregation function returns a scalar
        # with no tolerance for missing data values and turns it into a masked
        # array.
        axis = -1
        data = self.array.flatten()
        mock_return = 2
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0)
            self.assertMaskedArrayEqual(result, ma.array(2, mask=True))
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_non_masked_array_from_masked_array(self):
        # Providing a masked array, call_func returning a non-masked array,
        # resulting in a masked array output.
        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        result_axis_0 = ma.array(mock_return, mask=[True, True, False])
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=0.45)
            self.assertMaskedArrayAlmostEqual(result, result_axis_0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with mock.patch.object(self.TEST,
                               "call_func",
                               return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=0.45)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_kwarg_pass_through_no_kwargs(self):
        call_func = mock.Mock()
        data = mock.sentinel.data
        axis = mock.sentinel.axis
        aggregator = Aggregator("", call_func)
        aggregator.aggregate(data, axis)
        call_func.assert_called_once_with(data, axis=axis)

    def test_kwarg_pass_through_call_kwargs(self):
        call_func = mock.Mock()
        data = mock.sentinel.data
        axis = mock.sentinel.axis
        kwargs = dict(wibble="wobble", foo="bar")
        aggregator = Aggregator("", call_func)
        aggregator.aggregate(data, axis, **kwargs)
        call_func.assert_called_once_with(data, axis=axis, **kwargs)

    def test_kwarg_pass_through_init_kwargs(self):
        call_func = mock.Mock()
        data = mock.sentinel.data
        axis = mock.sentinel.axis
        kwargs = dict(wibble="wobble", foo="bar")
        aggregator = Aggregator("", call_func, **kwargs)
        aggregator.aggregate(data, axis)
        call_func.assert_called_once_with(data, axis=axis, **kwargs)

    def test_kwarg_pass_through_combined_kwargs(self):
        call_func = mock.Mock()
        data = mock.sentinel.data
        axis = mock.sentinel.axis
        init_kwargs = dict(wibble="wobble", var=1.0)
        call_kwargs = dict(foo="foo", var=0.5)
        aggregator = Aggregator("", call_func, **init_kwargs)
        aggregator.aggregate(data, axis, **call_kwargs)
        expected_kwargs = init_kwargs.copy()
        expected_kwargs.update(call_kwargs)
        call_func.assert_called_once_with(data, axis=axis, **expected_kwargs)

    def test_mdtol_intercept(self):
        call_func = mock.Mock()
        data = mock.sentinel.data
        axis = mock.sentinel.axis
        aggregator = Aggregator("", call_func)
        aggregator.aggregate(data, axis, wibble="wobble", mdtol=0.8)
        call_func.assert_called_once_with(data, axis=axis, wibble="wobble")

    def test_no_lazy_func(self):
        dummy_agg = Aggregator("custom_op", lambda x: 1)
        expected = "custom_op aggregator does not support lazy operation"
        with self.assertRaisesRegex(LazyAggregatorError, expected):
            dummy_agg.lazy_aggregate(np.arange(10), axis=0)
예제 #37
0
    def process(self, cube, weights=None):
        """Calculate weighted blend across the chosen coord, for either
           probabilistic or percentile data. If there is a percentile
           coordinate on the cube, it will blend using the
           PercentileBlendingAggregator but the percentile coordinate must
           have at least two points.

        Args:
            cube (iris.cube.Cube):
                   Cube to blend across the coord.
            weights (Optional list or np.array of weights):
                     or None (equivalent to equal weights).

        Returns:
            result (iris.cube.Cube):
                     containing the weighted blend across the chosen coord.

        Raises:
            TypeError : If the first argument not a cube.
            ValueError : If there is a percentile coord and it is not a
                           dimension coord in the cube.
            ValueError : If there is a percentile dimension with only one
                            point, we need at least two points in order to do
                            the blending.
            ValueError : If there are more than one percentile coords
                           in the cube.
            ValueError : If there is a percentile dimension on the cube and the
                         mode for blending is 'weighted_maximum'
            ValueError : If the weights shape do not match the dimension
                           of the coord we are blending over.
        Warns:
            Warning : If trying to blend across a scalar coordinate with only
                        one value. Returns the original cube in this case.

        """
        if not isinstance(cube, iris.cube.Cube):
            msg = ('The first argument must be an instance of '
                   'iris.cube.Cube but is'
                   ' {0:s}.'.format(type(cube)))
            raise TypeError(msg)

        # Check to see if the data is percentile data

        try:
            perc_coord = find_percentile_coordinate(cube)
            perc_dim = cube.coord_dims(perc_coord.name())
            if not perc_dim:
                msg = ('The percentile coord must be a dimension '
                       'of the cube.')
                raise ValueError(msg)
            # Check the percentile coordinate has more than one point,
            # otherwise raise an error as we won't be able to blend.
            if len(perc_coord.points) < 2.0:
                msg = ('Percentile coordinate does not have enough points'
                       ' in order to blend. Must have at least 2 percentiles.')
                raise ValueError(msg)
        except CoordinateNotFoundError:
            perc_coord = None
            perc_dim = None

        # If we have a percentile dimension and the mode is 'max' raise an
        # exception.
        if perc_coord and self.mode == 'weighted_maximum':
            msg = ('The "weighted_maximum" mode cannot be used with'
                   ' percentile data.')
            raise ValueError(msg)

        # check weights array matches coordinate shape if not None
        if weights is not None:
            if np.array(weights).shape != cube.coord(self.coord).points.shape:
                msg = ('The weights array must match the shape '
                       'of the coordinate in the input cube; '
                       'weight shape is '
                       '{0:s}'.format(np.array(weights).shape) +
                       ', cube shape is '
                       '{0:s}'.format(cube.coord(self.coord).points.shape))
                raise ValueError(msg)

        # If coord to blend over is a scalar_coord warn
        # and return original cube.
        coord_dim = cube.coord_dims(self.coord)
        if not coord_dim:
            msg = ('Trying to blend across a scalar coordinate with only one'
                   ' value. Returning original cube')
            warnings.warn(msg)
            result = cube
        else:
            try:
                cube.coord('threshold')
            except iris.exceptions.CoordinateNotFoundError:
                slices_over_threshold = [cube]
            else:
                if self.coord != 'threshold':
                    slices_over_threshold = cube.slices_over('threshold')
                else:
                    slices_over_threshold = [cube]

            cubelist = iris.cube.CubeList([])
            for cube_thres in slices_over_threshold:
                # Blend the cube across the coordinate
                # Use percentile Aggregator if required
                if perc_coord and self.mode == "weighted_mean":
                    percentiles = np.array(perc_coord.points, dtype=float)
                    perc_dim, = cube_thres.coord_dims(perc_coord.name())
                    # Set equal weights if none are provided
                    if weights is None:
                        num = len(cube_thres.coord(self.coord).points)
                        weights = np.ones(num) / float(num)
                    # Set up aggregator
                    PERCENTILE_BLEND = (Aggregator(
                        'weighted_mean',
                        PercentileBlendingAggregator.aggregate))

                    cube_new = cube_thres.collapsed(self.coord,
                                                    PERCENTILE_BLEND,
                                                    arr_percent=percentiles,
                                                    arr_weights=weights,
                                                    perc_dim=perc_dim)

                # Else do a simple weighted average
                elif self.mode == "weighted_mean":
                    # Equal weights are used as default.
                    weights_array = None
                    # Else broadcast the weights to be used by the aggregator.
                    coord_dim_thres = cube_thres.coord_dims(self.coord)
                    if weights is not None:
                        weights_array = (iris.util.broadcast_to_shape(
                            np.array(weights), cube_thres.shape,
                            coord_dim_thres))
                    orig_cell_methods = cube_thres.cell_methods
                    # Calculate the weighted average.
                    cube_new = cube_thres.collapsed(self.coord,
                                                    iris.analysis.MEAN,
                                                    weights=weights_array)
                    # Update the name of the cell_method created by Iris to
                    # 'weighted_mean' to be consistent.
                    new_cell_methods = cube_new.cell_methods
                    extra_cm = (set(new_cell_methods) -
                                set(orig_cell_methods)).pop()
                    add_renamed_cell_method(cube_new, extra_cm,
                                            'weighted_mean')

                # Else use the maximum probability aggregator.
                elif self.mode == "weighted_maximum":
                    # Set equal weights if none are provided
                    if weights is None:
                        num = len(cube_thres.coord(self.coord).points)
                        weights = np.ones(num) / float(num)
                    # Set up aggregator
                    MAX_PROBABILITY = (Aggregator(
                        'weighted_maximum',
                        MaxProbabilityAggregator.aggregate))

                    cube_new = cube_thres.collapsed(self.coord,
                                                    MAX_PROBABILITY,
                                                    arr_weights=weights)
                cubelist.append(cube_new)
            result = cubelist.merge_cube()
            if isinstance(cubelist[0].data, np.ma.core.MaskedArray):
                result.data = np.ma.array(result.data)
        # If set adjust values of collapsed coordinates.
        if self.coord_adjust is not None:
            for crd in result.coords():
                if cube.coord_dims(crd.name()) == coord_dim:
                    pnts = cube.coord(crd.name()).points
                    crd.points = np.array(self.coord_adjust(pnts),
                                          dtype=crd.points.dtype)

        return result
예제 #38
0
 def test_no_lazy_func(self):
     dummy_agg = Aggregator("custom_op", lambda x: 1)
     expected = "custom_op aggregator does not support lazy operation"
     with self.assertRaisesRegex(LazyAggregatorError, expected):
         dummy_agg.lazy_aggregate(np.arange(10), axis=0)
예제 #39
0
class Test_aggregate(tests.IrisTest):
    # These unit tests don't call a data aggregation function, they call a
    # mocked one i.e. the return values of the mocked data aggregation
    # function don't matter, only how these are dealt with by the aggregate
    # method.
    def setUp(self):
        self.TEST = Aggregator('test', None)
        self.array = ma.array([[1, 2, 3],
                               [4, 5, 6]],
                              mask=[[False, True, False],
                                    [True, False, False]],
                              dtype=np.float64)
        self.expected_result_axis0 = ma.array([1, 2, 3], mask=None)
        self.expected_result_axis1 = ma.array([4, 5], mask=None)

    def test_masked_notol(self):
        # Providing masked array with no tolerance keyword (mdtol) provided.
        axis = 0
        mock_return = self.expected_result_axis0.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis)
            self.assertMaskedArrayEqual(result, self.expected_result_axis0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_above_tol(self):
        # Providing masked array with a high tolerance (mdtol) provided.
        axis = 0
        mock_return = self.expected_result_axis0.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.55)
            self.assertMaskedArrayEqual(result, self.expected_result_axis0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.55)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_below_tol(self):
        # Providing masked array with a tolerance on missing values, low
        # enough to modify the resulting mask for axis 0.
        axis = 0
        result_axis_0 = self.expected_result_axis0.copy()
        result_axis_0.mask = np.array([True, True, False])
        mock_return = ma.array([1, 2, 3], mask=None)
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayAlmostEqual(result, result_axis_0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_below_tol_alt(self):
        # Providing masked array with a tolerance on missing values, low
        # enough to modify the resulting mask for axis 1.
        axis = 1
        result_axis_1 = self.expected_result_axis1.copy()
        result_axis_1.mask = np.array([True, True])
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.1)
            self.assertMaskedArrayAlmostEqual(result, result_axis_1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_unmasked_with_mdtol(self):
        # Providing aggregator with an unmasked array and tolerance specified
        # for missing data - ensure that result is unaffected.
        data = self.array.data

        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0.5)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0.5)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

    def test_unmasked(self):
        # Providing aggregator with an unmasked array and no additional keyword
        # arguments ensure that result is unaffected.
        data = self.array.data

        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_array_len_one_mdtol(self):
        # Test the case when the data aggregation function returns a scalar and
        # turns it into a masked array.
        axis = -1
        data = self.array.flatten()
        mock_return = 2
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=1)
            self.assertMaskedArrayEqual(result, ma.array([2], mask=[False]))
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_array_len_one_mdtol_alt(self):
        # Test the case when the data aggregation function returns a scalar
        # with no tolerance for missing data values and turns it into a masked
        # array of length one.
        axis = -1
        data = self.array.flatten()
        mock_return = 2
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0)
            self.assertMaskedArrayEqual(result, ma.array([2], mask=[True]))
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_non_masked_array_from_masked_array(self):
        # Providing a masked array, call_func returning a non-masked array,
        # resulting in a masked array output.
        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        result_axis_0 = ma.array(mock_return, mask=[True, True, False])
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayAlmostEqual(result, result_axis_0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)
예제 #40
0
class Test_aggregate(tests.IrisTest):
    # These unit tests don't call a data aggregation function, they call a
    # mocked one i.e. the return values of the mocked data aggregation
    # function don't matter, only how these are dealt with by the aggregate
    # method.
    def setUp(self):
        self.TEST = Aggregator('test', None)
        self.array = ma.array([[1, 2, 3],
                               [4, 5, 6]],
                              mask=[[False, True, False],
                                    [True, False, False]],
                              dtype=np.float64)
        self.expected_result_axis0 = ma.array([1, 2, 3], mask=None)
        self.expected_result_axis1 = ma.array([4, 5], mask=None)

    def test_masked_notol(self):
        # Providing masked array with no tolerance keyword (mdtol) provided.
        axis = 0
        mock_return = self.expected_result_axis0.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis)
            self.assertMaskedArrayEqual(result, self.expected_result_axis0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_above_tol(self):
        # Providing masked array with a high tolerance (mdtol) provided.
        axis = 0
        mock_return = self.expected_result_axis0.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.55)
            self.assertMaskedArrayEqual(result, self.expected_result_axis0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.55)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_below_tol(self):
        # Providing masked array with a tolerance on missing values, low
        # enough to modify the resulting mask for axis 0.
        axis = 0
        result_axis_0 = self.expected_result_axis0.copy()
        result_axis_0.mask = np.array([True, True, False])
        mock_return = ma.array([1, 2, 3], mask=None)
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayAlmostEqual(result, result_axis_0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_masked_below_tol_alt(self):
        # Providing masked array with a tolerance on missing values, low
        # enough to modify the resulting mask for axis 1.
        axis = 1
        result_axis_1 = self.expected_result_axis1.copy()
        result_axis_1.mask = np.array([True, True])
        mock_return = self.expected_result_axis1.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.1)
            self.assertMaskedArrayAlmostEqual(result, result_axis_1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_unmasked_with_mdtol(self):
        # Providing aggregator with an unmasked array and tolerance specified
        # for missing data - ensure that result is unaffected.
        data = self.array.data

        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0.5)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0.5)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

    def test_unmasked(self):
        # Providing aggregator with an unmasked array and no additional keyword
        # arguments ensure that result is unaffected.
        data = self.array.data

        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis)
            self.assertArrayAlmostEqual(result, mock_return.copy())
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_scalar_mdtol(self):
        # Test the case when the data aggregation function returns a scalar and
        # turns it into a masked array.
        axis = -1
        data = self.array.flatten()
        mock_return = 2
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=1)
            self.assertMaskedArrayEqual(result, ma.array(2, mask=False))
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_scalar_mdtol_alt(self):
        # Test the case when the data aggregation function returns a scalar
        # with no tolerance for missing data values and turns it into a masked
        # array.
        axis = -1
        data = self.array.flatten()
        mock_return = 2
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(data, axis, mdtol=0)
            self.assertMaskedArrayEqual(result, ma.array(2, mask=True))
        mock_method.assert_called_once_with(data, axis=axis)

    def test_returning_non_masked_array_from_masked_array(self):
        # Providing a masked array, call_func returning a non-masked array,
        # resulting in a masked array output.
        axis = 0
        mock_return = self.expected_result_axis0.data.copy()
        result_axis_0 = ma.array(mock_return, mask=[True, True, False])
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayAlmostEqual(result, result_axis_0)
        mock_method.assert_called_once_with(self.array, axis=axis)

        axis = 1
        mock_return = self.expected_result_axis1.data.copy()
        with patch.object(self.TEST, 'call_func',
                          return_value=mock_return) as mock_method:
            result = self.TEST.aggregate(self.array, axis, mdtol=.45)
            self.assertMaskedArrayEqual(result, self.expected_result_axis1)
        mock_method.assert_called_once_with(self.array, axis=axis)

    def test_kwarg_pass_through_no_kwargs(self):
        call_func = Mock()
        data = sentinel.data
        axis = sentinel.axis
        aggregator = Aggregator('', call_func)
        aggregator.aggregate(data, axis)
        call_func.assert_called_once_with(data, axis=axis)

    def test_kwarg_pass_through_call_kwargs(self):
        call_func = Mock()
        data = sentinel.data
        axis = sentinel.axis
        kwargs = dict(wibble='wobble', foo='bar')
        aggregator = Aggregator('', call_func)
        aggregator.aggregate(data, axis, **kwargs)
        call_func.assert_called_once_with(data, axis=axis, **kwargs)

    def test_kwarg_pass_through_init_kwargs(self):
        call_func = Mock()
        data = sentinel.data
        axis = sentinel.axis
        kwargs = dict(wibble='wobble', foo='bar')
        aggregator = Aggregator('', call_func, **kwargs)
        aggregator.aggregate(data, axis)
        call_func.assert_called_once_with(data, axis=axis, **kwargs)

    def test_kwarg_pass_through_combined_kwargs(self):
        call_func = Mock()
        data = sentinel.data
        axis = sentinel.axis
        init_kwargs = dict(wibble='wobble', var=1.0)
        call_kwargs = dict(foo='foo', var=0.5)
        aggregator = Aggregator('', call_func, **init_kwargs)
        aggregator.aggregate(data, axis, **call_kwargs)
        expected_kwargs = init_kwargs.copy()
        expected_kwargs.update(call_kwargs)
        call_func.assert_called_once_with(data, axis=axis, **expected_kwargs)

    def test_mdtol_intercept(self):
        call_func = Mock()
        data = sentinel.data
        axis = sentinel.axis
        aggregator = Aggregator('', call_func)
        aggregator.aggregate(data, axis, wibble='wobble', mdtol=0.8)
        call_func.assert_called_once_with(data, axis=axis, wibble='wobble')

    def test_no_lazy_func(self):
        dummy_agg = Aggregator('custom_op', lambda x: 1)
        expected = 'custom_op aggregator does not support lazy operation'
        with self.assertRaisesRegexp(LazyAggregatorError, expected):
            dummy_agg.lazy_aggregate(np.arange(10), axis=0)
예제 #41
0
 def test_no_lazy_func(self):
     dummy_agg = Aggregator('custom_op', lambda x: 1)
     expected = 'custom_op aggregator does not support lazy operation'
     with self.assertRaisesRegexp(LazyAggregatorError, expected):
         dummy_agg.lazy_aggregate(np.arange(10), axis=0)
예제 #42
0
def consecutive_dry_days(cube, period='year', length=6, threshold=1.):
    """
    calculate consecutive dry days within an iris.cube.Cube
    Args:
    * cube (iris.cube.Cube):
        An iris.cube.Cube holding precipiation amount in mm/day
    * period (string):
        Period over that the CDD will be calculated. Can be 'year', 'season'
        or 'month'. If period is 'season' or 'month' the CDD will be averaged
        over the years
    Kwargs:
    * length (int):
        The number of days without rainfall that define a dry period
    * threshold (float):
        The upper limit of daily rainfall in mm that indicates
        'no precipitation'
    Returns:
        An iris.cube.CubeList that holds two iris.cube.Cubes with the longest
        period of dry days in the given period and the mean of the number of
        dry periods with respect to the given length
    """
    def _cdd_index(array, axis, threshold):
        """
        Calculate the consecutive dry days index.
        This function is used as an iris.analysis.Aggregator
        Args:
        * array (numpy.array or numpy.ma.array):
            array that holds the precipitation data
        * axis (int):
            the number of the time-axis
        * threshold (float):
            the threshold that indicates a precipiation-less day
        Returns:
            the aggregation result, collapsing the 'axis' dimension of
            the 'data' argument
        """
        from pycat.analysis.utils import (_get_max_true_block_length,
                                          _get_true_block_lengths)

        up_down = _get_true_block_lengths(array < threshold, axis)
        return _get_max_true_block_length(up_down)

    def _cdd_periods(array, axis, threshold, length):
        """
        Calculate the number of consecutive dry days periods.
        This function is used as an iris.analysis.Aggregator
        Args:
        * array (numpy.array or numpy.ma.array):
            array that holds the precipitation data
        * axis (int):
            the number of the time-axis
        * threshold (float):
            the threshold that indicates a precipiation-less day
        * length (int):
            number of days that a dry period must last
        Returns:
            the aggregation result, collapsing the 'axis' dimension
            of the 'data' argument
        """
        from pycat.analysis.utils import (_get_len_true_block_length,
                                          _get_true_block_lengths)

        up_down = _get_true_block_lengths(array < threshold, axis)
        return _get_len_true_block_length(up_down, length)

    # build the iris.analysis.Aggregators
    cdd_index = Aggregator('cdd_index', _cdd_index)
    cdd_periods = Aggregator('cdd_periods', _cdd_periods)

    # check if the cube already has the needed auxiliary coordinates
    if period == 'season':
        # add the season_year auxiliary coordinate
        try:
            years = np.unique(cube.coord('season_year').points)
        except CoordinateNotFoundError:
            ccat.add_season_year(cube, 'time')
            years = np.unique(cube.coord('season_year').points)
        constraint_year_key = 'season_year'
    else:
        # add calendar years
        try:
            years = np.unique(cube.coord('year').points)
        except CoordinateNotFoundError:
            ccat.add_year(cube, 'time')
            years = np.unique(cube.coord('year').points)
        constraint_year_key = 'year'

    if period in ['season', 'month']:
        try:
            index_period = np.unique(cube.coord('%s_number' % period).points)
        except CoordinateNotFoundError:
            cat = getattr(ccat, 'add_%s_number' % period)
            cat(cube, 'time')
            index_period = np.unique(cube.coord('%s_number' % period).points)

    # create time-axis of resulting cubes
    time_dimension = _make_time_dimension(
        cube.coord('time').units.num2date(cube.coord('time').points[0]),
        cube.coord('time').units.num2date(cube.coord('time').points[-1]),
        period=period)
    # create the empty resulting cubes
    dim_coords_and_dims = []
    slices = []
    for coord in cube.dim_coords:
        if coord.units.is_time_reference():
            dim_coords_and_dims.append(
                (time_dimension, cube.coord_dims(coord)))
            slices.append(0)
            time_axis = cube.coord_dims(coord)[0]
        else:
            dim_coords_and_dims.append((coord, cube.coord_dims(coord)))
            slices.append(slice(None, None, None))

    cdd_index_cube = _create_cube(
        long_name='Consecutive dry days is the greatest number of '
        'consecutive days per time period with daily '
        'precipitation amount below %s mm.' % threshold,
        var_name='consecutive_dry_days_index_per_time_period',
        units=iris.unit.Unit('1'),
        dim_coords_and_dims=dim_coords_and_dims)

    cdd_periods_cube = _create_cube(
        long_name='Number of cdd periods in given time period '
        'with more than %d days.' % length,
        var_name='number_of_cdd_periods_with_more_than_'
        '%ddays_per_time_period' % length,
        units=iris.unit.Unit('1'),
        dim_coords_and_dims=dim_coords_and_dims)

    # differentiate between the considered period
    if period == 'year':
        # just run the aggregation over all given years resulting in
        # the maximum cdd length and the number of cdd periods for each year
        for year in years:
            tmp_cube = cube.extract(iris.Constraint(year=year))
            slices[time_axis] = year - years[0]
            cdd_index_data = tmp_cube.collapsed('time',
                                                cdd_index,
                                                threshold=threshold).data
            cdd_periods_data = tmp_cube.collapsed('time',
                                                  cdd_periods,
                                                  threshold=threshold,
                                                  length=length).data

            cdd_index_cube.data[slices] = cdd_index_data
            cdd_periods_cube.data[slices] = cdd_periods_data

        return iris.cube.CubeList((cdd_index_cube, cdd_periods_cube))

    else:
        # run the aggregation over all seasons/months of all years
        # afterwards aggregate the seasons/month by MAX Aggregator
        # for the cdd_index and the MEAN Aggregator for cdd_periods
        for year in years:
            for p in index_period:
                constraint_dict = {
                    '%s_number' % period: p,
                    constraint_year_key: year
                }
                tmp_cube = cube.extract(iris.Constraint(**constraint_dict))
                if tmp_cube:
                    # the extraction can lead to empty cubes for seasons
                    # in the last year
                    time_index = (year - years[0]) * len(index_period) + p
                    # months numbers start at 1
                    if period == 'month':
                        time_index -= 1
                    slices[time_axis] = time_index
                    cdd_index_data = tmp_cube.collapsed(
                        'time', cdd_index, threshold=threshold).data
                    cdd_periods_data = tmp_cube.collapsed('time',
                                                          cdd_periods,
                                                          threshold=threshold,
                                                          length=length).data

                    cdd_index_cube.data[slices] = cdd_index_data
                    cdd_periods_cube.data[slices] = cdd_periods_data

        # aggregate over seasons/months
        cat = getattr(ccat, 'add_%s' % period)
        cat(cdd_index_cube, 'time')
        cat(cdd_periods_cube, 'time')

        cdd_index_mean = cdd_index_cube.aggregated_by(period,
                                                      iris.analysis.MEAN)
        cdd_periods_mean = cdd_periods_cube.aggregated_by(
            period, iris.analysis.MEAN)

        cdd_index_mean.remove_coord('time')
        cdd_periods_mean.remove_coord('time')

        return iris.cube.CubeList((cdd_index_mean, cdd_periods_mean))