def test_model_id_attr_mismatch(self): """Test that when a model ID attribute string is specified that does not match the model ID attribute key name on both cubes to be merged, an error is thrown""" plugin = MergeCubesForWeightedBlending( "model", model_id_attr="non_matching_model_config") msg = "Cannot create model ID coordinate" with self.assertRaisesRegex(ValueError, msg): plugin.process(self.non_mo_cubelist)
def test_model_id_attr_mismatch_one_cube(self): """Test that when a model ID attribute string is specified that only matches the model ID attribute key name on one of the cubes to be merged, an error is thrown""" self.non_mo_cubelist[1].attributes.pop("non_mo_model_config") self.non_mo_cubelist[1].attributes[ "non_matching_model_config"] = "non_uk_det" plugin = MergeCubesForWeightedBlending( "model", model_id_attr="non_matching_model_config") msg = "Cannot create model ID coordinate" with self.assertRaisesRegex(ValueError, msg): plugin.process(self.non_mo_cubelist)
def test_record_run_no_model_attr(self): """Test recording the source runs without a model attribute.""" plugin = MergeCubesForWeightedBlending( "model", weighting_coord="forecast_period", model_id_attr="mosg__model_configuration", record_run_attr="mosg__model_run", ) self.cube_ukv.attributes.pop("mosg__model_configuration") msg = "Failure to record run information" with self.assertRaisesRegex(Exception, msg): plugin.process([self.cube_ukv, self.cube_enuk])
def test_dict(self): """Test dictionary option for model blending with non-equal weights""" data = np.ones((3, 3, 3), dtype=np.float32) thresholds = np.array([276, 277, 278], dtype=np.float32) ukv_cube = set_up_probability_cube(data, thresholds, time=dt(2018, 9, 10, 7), frt=dt(2018, 9, 10, 1), standard_grid_metadata="uk_det") enukx_cube = set_up_probability_cube(data, thresholds, time=dt(2018, 9, 10, 7), frt=dt(2018, 9, 10, 1), standard_grid_metadata="uk_ens") merger = MergeCubesForWeightedBlending( "model_id", weighting_coord="forecast_period", model_id_attr="mosg__model_configuration") cube = merger.process([ukv_cube, enukx_cube]) plugin = WeightAndBlend("model_id", "dict", weighting_coord="forecast_period", wts_dict=MODEL_WEIGHTS) # at 6 hours lead time we should have 1/3 UKV and 2/3 MOGREPS-UK, # according to the dictionary weights specified above weights = plugin._calculate_blending_weights(cube) self.assertArrayEqual( weights.coord("model_configuration").points, ["uk_det", "uk_ens"]) self.assertArrayAlmostEqual(weights.data, np.array([0.3333333, 0.6666667]))
def test_non_mo_model_id(self): """Test that a model ID attribute string can be specified when merging multi model cubes""" plugin = MergeCubesForWeightedBlending( "model", model_id_attr="non_mo_model_config") result = plugin.process(self.non_mo_cubelist) self.assertIsInstance(result, iris.cube.Cube) self.assertArrayEqual(result.coord("model_id").points, [0, 1000])
def setUp(self): """Set up cube and plugin""" cubelist = set_up_masked_cubes() merger = MergeCubesForWeightedBlending( "model_id", weighting_coord="forecast_period", model_id_attr="mosg__model_configuration") self.cube = merger.process(cubelist) self.plugin = WeightAndBlend( "model_id", "dict", weighting_coord="forecast_period", wts_dict=MODEL_WEIGHTS) self.initial_weights = ( self.plugin._calculate_blending_weights(self.cube))
def test_record_run(self): """Test recording the source runs in a blend record run attribute.""" plugin = MergeCubesForWeightedBlending( "model", weighting_coord="forecast_period", model_id_attr="mosg__model_configuration", record_run_attr="mosg__model_run", ) cube = plugin.process(self.cubelist) self.assertEqual( cube.attributes["mosg__model_run"], "uk_det:20151123T0300Z:\nuk_ens:20151123T0000Z:", )
def test_blend_realizations(self): """Test processing works for merging over coordinates that don't require specific setup""" data = np.ones((1, 3, 3), dtype=np.float32) cube1 = set_up_variable_cube(data, realizations=np.array([0])) cube1 = iris.util.squeeze(cube1) cube2 = set_up_variable_cube(data, realizations=np.array([1])) cube2 = iris.util.squeeze(cube2) plugin = MergeCubesForWeightedBlending("realization") result = plugin.process([cube1, cube2]) self.assertIsInstance(result, iris.cube.Cube) self.assertArrayEqual(result.coord("realization").points, np.array([0, 1])) self.assertEqual(result[0].metadata, cube1.metadata) self.assertEqual(result[1].metadata, cube2.metadata)
def test_record_run_existing(self): """Test recording blend source runs with existing record attributes.""" plugin = MergeCubesForWeightedBlending( "model", weighting_coord="forecast_period", model_id_attr="mosg__model_configuration", record_run_attr="mosg__model_run", ) self.cube_ukv.attributes[ "mosg__model_run"] = "uk_det:20151123T0200Z:\nuk_det:20151123T0300Z:" cube = plugin.process([self.cube_ukv, self.cube_enuk]) self.assertEqual( cube.attributes["mosg__model_run"], "uk_det:20151123T0200Z:\nuk_det:20151123T0300Z:\nuk_ens:20151123T0000Z:", )
def test_cycle_blend(self): """Test merge for blending over forecast_reference_time""" cube = self.cube_ukv.copy() cube.coord("forecast_reference_time").points = ( cube.coord("forecast_reference_time").points + 3600) cube.coord("forecast_period").points = ( cube.coord("forecast_reference_time").points - 3600) plugin = MergeCubesForWeightedBlending("forecast_reference_time") result = plugin.process([self.cube_ukv, cube]) self.assertIsInstance(result, iris.cube.Cube) self.assertIn(result.coord("forecast_reference_time"), result.coords(dim_coords=True)) # check no model coordinates have been added with self.assertRaises(iris.exceptions.CoordinateNotFoundError): result.coord(MODEL_BLEND_COORD) with self.assertRaises(iris.exceptions.CoordinateNotFoundError): result.coord(MODEL_NAME_COORD)
def test_blend_coord_ascending(self): """Test the order of the output blend coordinate is always ascending, independent of the input cube order""" frt = self.cube_ukv.coord("forecast_reference_time").points[0] fp = self.cube_ukv.coord("forecast_period").points[0] cube1 = self.cube_ukv.copy() cube1.coord("forecast_reference_time").points = [frt + 3600] cube1.coord("forecast_period").points = [fp - 3600] cube2 = self.cube_ukv.copy() cube2.coord("forecast_reference_time").points = [frt + 7200] cube2.coord("forecast_period").points = [fp - 7200] # input unordered cubes; expect ordered output expected_points = np.array([frt, frt + 3600, frt + 7200], dtype=np.int64) plugin = MergeCubesForWeightedBlending("forecast_reference_time") result = plugin.process([cube1, self.cube_ukv, cube2]) self.assertArrayEqual( result.coord("forecast_reference_time").points, expected_points)
def main(argv=None): """Load in arguments and ensure they are set correctly. Then load in the data to blend and calculate default weights using the method chosen before carrying out the blending.""" parser = ArgParser( description='Calculate the default weights to apply in weighted ' 'blending plugins using the ChooseDefaultWeightsLinear or ' 'ChooseDefaultWeightsNonLinear plugins. Then apply these ' 'weights to the dataset using the BasicWeightedAverage plugin.' ' Required for ChooseDefaultWeightsLinear: y0val and ynval.' ' Required for ChooseDefaultWeightsNonLinear: cval.' ' Required for ChooseWeightsLinear with dict: wts_dict.') parser.add_argument('--wts_calc_method', metavar='WEIGHTS_CALCULATION_METHOD', choices=['linear', 'nonlinear', 'dict'], default='linear', help='Method to use to calculate ' 'weights used in blending. "linear" (default): ' 'calculate linearly varying blending weights. ' '"nonlinear": calculate blending weights that decrease' ' exponentially with increasing blending coordinate. ' '"dict": calculate weights using a dictionary passed ' 'in as a command line argument.') parser.add_argument('coordinate', type=str, metavar='COORDINATE_TO_AVERAGE_OVER', help='The coordinate over which the blending ' 'will be applied.') parser.add_argument('--coordinate_unit', metavar='UNIT_STRING', default='hours since 1970-01-01 00:00:00', help='Units for blending coordinate. Default= ' 'hours since 1970-01-01 00:00:00') parser.add_argument('--calendar', metavar='CALENDAR', help='Calendar for time coordinate. Default=gregorian') parser.add_argument('--cycletime', metavar='CYCLETIME', type=str, help='The forecast reference time to be used after ' 'blending has been applied, in the format ' 'YYYYMMDDTHHMMZ. If not provided, the blended file ' 'will take the latest available forecast reference ' 'time from the input cubes supplied.') parser.add_argument('--model_id_attr', metavar='MODEL_ID_ATTR', type=str, default="mosg__model_configuration", help='The name of the netCDF file attribute to be ' 'used to identify the source model for ' 'multi-model blends. Default assumes Met Office ' 'model metadata. Must be present on all input ' 'files if blending over models.') parser.add_argument('--spatial_weights_from_mask', action='store_true', default=False, help='If set this option will result in the generation' ' of spatially varying weights based on the' ' masks of the data we are blending. The' ' one dimensional weights are first calculated ' ' using the chosen weights calculation method,' ' but the weights will then be adjusted spatially' ' based on where there is masked data in the data' ' we are blending. The spatial weights are' ' calculated using the' ' SpatiallyVaryingWeightsFromMask plugin.') parser.add_argument('weighting_mode', metavar='WEIGHTED_BLEND_MODE', choices=['weighted_mean', 'weighted_maximum'], help='The method used in the weighted blend. ' '"weighted_mean": calculate a normal weighted' ' mean across the coordinate. ' '"weighted_maximum": multiplies the values in the' ' coordinate by the weights, and then takes the' ' maximum.') parser.add_argument('input_filepaths', metavar='INPUT_FILES', nargs="+", help='Paths to input files to be blended.') parser.add_argument('output_filepath', metavar='OUTPUT_FILE', help='The output path for the processed NetCDF.') spatial = parser.add_argument_group( 'Spatial weights from mask options', 'Options for calculating the spatial weights using the ' 'SpatiallyVaryingWeightsFromMask plugin.') spatial.add_argument('--fuzzy_length', metavar='FUZZY_LENGTH', type=float, default=20000, help='When calculating spatially varying weights we' ' can smooth the weights so that areas close to' ' areas that are masked have lower weights than' ' those further away. This fuzzy length controls' ' the scale over which the weights are smoothed.' ' The fuzzy length is in terms of m, the' ' default is 20km. This distance is then' ' converted into a number of grid squares,' ' which does not have to be an integer. Assumes' ' the grid spacing is the same in the x and y' ' directions, and raises an error if this is not' ' true. See SpatiallyVaryingWeightsFromMask for' ' more detail.') linear = parser.add_argument_group( 'linear weights options', 'Options for the linear weights ' 'calculation in ' 'ChooseDefaultWeightsLinear') linear.add_argument('--y0val', metavar='LINEAR_STARTING_POINT', type=float, help='The relative value of the weighting start point ' '(lowest value of blend coord) for choosing default ' 'linear weights. This must be a positive float or 0.') linear.add_argument('--ynval', metavar='LINEAR_END_POINT', type=float, help='The relative value of the weighting ' 'end point (highest value of blend coord) for choosing' ' default linear weights. This must be a positive ' 'float or 0. Note that if blending over forecast ' 'reference time, ynval >= y0val would normally be ' 'expected (to give greater weight to the more recent ' 'forecast).') nonlinear = parser.add_argument_group( 'nonlinear weights options', 'Options for the non-linear ' 'weights calculation in ' 'ChooseDefaultWeightsNonLinear') nonlinear.add_argument('--cval', metavar='NON_LINEAR_FACTOR', type=float, help='Factor used to determine how skewed the ' 'non linear weights will be. ' 'A value of 1 implies equal weighting. If not ' 'set, a default value of cval=0.85 is set.') wts_dict = parser.add_argument_group( 'dict weights options', 'Options for linear weights to be ' 'calculated based on parameters ' 'read from a json file dict') wts_dict.add_argument('--wts_dict', metavar='WEIGHTS_DICTIONARY', help='Path to json file containing dictionary from ' 'which to calculate blending weights. Dictionary ' 'format is as specified in the improver.blending.' 'weights.ChooseWeightsLinear plugin.') wts_dict.add_argument('--weighting_coord', metavar='WEIGHTING_COORD', default='forecast_period', help='Name of ' 'coordinate over which linear weights should be ' 'scaled. This coordinate must be avilable in the ' 'weights dictionary.') args = parser.parse_args(args=argv) # if the linear weights method is called with non-linear args or vice # versa, exit with error if (args.wts_calc_method == "linear") and args.cval: parser.wrong_args_error('cval', 'linear') if ((args.wts_calc_method == "nonlinear") and np.any([args.y0val, args.ynval])): parser.wrong_args_error('y0val, ynval', 'non-linear') if (args.wts_calc_method == "dict") and not args.wts_dict: parser.error('Dictionary is required if --wts_calc_method="dict"') # set blending coordinate units if "time" in args.coordinate: coord_unit = Unit(args.coordinate_unit, args.calendar) elif args.coordinate_unit != 'hours since 1970-01-01 00:00:00.': coord_unit = args.coordinate_unit else: coord_unit = 'no_unit' # For blending across models, only blending across "model_id" is directly # supported. This is because the blending coordinate must be sortable, in # order to ensure that the data cube and the weights cube have coordinates # in the same order for blending. Whilst the model_configuration is # sortable itself, as it is associated with model_id, which is the # dimension coordinate, sorting the model_configuration coordinate can # result in the model_id coordinate becoming non-monotonic. As dimension # coordinates must be monotonic, this leads to the model_id coordinate # being demoted to an auxiliary coordinate. Therefore, for simplicity # model_id is used as the blending coordinate, instead of # model_configuration. # TODO: Support model_configuration as a blending coordinate directly. if args.coordinate == "model_configuration": blend_coord = "model_id" dict_coord = "model_configuration" else: blend_coord = args.coordinate dict_coord = args.coordinate # load cubes to be blended cubelist = load_cubelist(args.input_filepaths) # determine whether or not to equalise forecast periods for model # blending weights calculation weighting_coord = (args.weighting_coord if args.weighting_coord else "forecast_period") # prepare cubes for weighted blending merger = MergeCubesForWeightedBlending(blend_coord, weighting_coord=weighting_coord, model_id_attr=args.model_id_attr) cube = merger.process(cubelist, cycletime=args.cycletime) # if the coord for blending does not exist or has only one value, # update metadata only coord_names = [coord.name() for coord in cube.coords()] if (blend_coord not in coord_names) or (len( cube.coord(blend_coord).points) == 1): result = cube.copy() conform_metadata(result, cube, blend_coord, cycletime=args.cycletime) # raise a warning if this happened because the blend coordinate # doesn't exist if blend_coord not in coord_names: warnings.warn('Blend coordinate {} is not present on input ' 'data'.format(blend_coord)) # otherwise, calculate weights and blend across specified dimension else: weights = calculate_blending_weights( cube, blend_coord, args.wts_calc_method, wts_dict=args.wts_dict, weighting_coord=args.weighting_coord, coord_unit=coord_unit, y0val=args.y0val, ynval=args.ynval, cval=args.cval, dict_coord=dict_coord) if args.spatial_weights_from_mask: check_if_grid_is_equal_area(cube) grid_cells_x, _ = convert_distance_into_number_of_grid_cells( cube, args.fuzzy_length, int_grid_cells=False) SpatialWeightsPlugin = SpatiallyVaryingWeightsFromMask( grid_cells_x) weights = SpatialWeightsPlugin.process(cube, weights, blend_coord) # blend across specified dimension BlendingPlugin = WeightedBlendAcrossWholeDimension( blend_coord, args.weighting_mode, cycletime=args.cycletime) result = BlendingPlugin.process(cube, weights=weights) save_netcdf(result, args.output_filepath)
class Test_process(IrisTest): """Test the process method""" def setUp(self): """Set up some probability cubes from different models""" data = np.array( [ 0.9 * np.ones((3, 3)), 0.5 * np.ones((3, 3)), 0.1 * np.ones( (3, 3)) ], dtype=np.float32, ) thresholds = np.array([273.0, 275.0, 277.0], dtype=np.float32) time_point = dt(2015, 11, 23, 7) time_bounds = [dt(2015, 11, 23, 4), time_point] # set up a MOGREPS-UK cube with 7 hour forecast period self.cube_enuk = set_up_probability_cube( data.copy(), thresholds, standard_grid_metadata="uk_ens", time=time_point, frt=dt(2015, 11, 23, 0), time_bounds=time_bounds, ) # set up a UKV cube with 4 hour forecast period self.cube_ukv = set_up_probability_cube( data.copy(), thresholds, standard_grid_metadata="uk_det", time=time_point, frt=dt(2015, 11, 23, 3), time_bounds=time_bounds, ) self.cubelist = iris.cube.CubeList([self.cube_enuk, self.cube_ukv]) # set up a nowcast cube self.cube_nowcast = set_up_probability_cube( data.copy(), thresholds, standard_grid_metadata="nc_det", time=time_point, frt=dt(2015, 11, 23, 3, 15), time_bounds=time_bounds, ) # set up some non-UK test cubes cube_non_mo_ens = self.cube_enuk.copy() cube_non_mo_ens.attributes.pop("mosg__model_configuration") cube_non_mo_ens.attributes["non_mo_model_config"] = "non_uk_ens" cube_non_mo_det = self.cube_ukv.copy() cube_non_mo_det.attributes.pop("mosg__model_configuration") cube_non_mo_det.attributes["non_mo_model_config"] = "non_uk_det" self.non_mo_cubelist = iris.cube.CubeList( [cube_non_mo_ens, cube_non_mo_det]) # set up plugin for multi-model blending weighted by forecast period self.plugin = MergeCubesForWeightedBlending( "model", weighting_coord="forecast_period", model_id_attr="mosg__model_configuration", ) def test_multi_model_merge(self): """Test models merge OK and have expected model coordinates""" result = self.plugin.process(self.cubelist) self.assertIsInstance(result, iris.cube.Cube) self.assertArrayEqual( result.coord(MODEL_BLEND_COORD).points, [0, 1000]) self.assertArrayEqual( result.coord(MODEL_NAME_COORD).points, ["uk_ens", "uk_det"]) def test_time_coords(self): """Test merged cube has scalar time coordinates if weighting models by forecast period""" result = self.plugin.process(self.cubelist) # test resulting cube has single 4 hour (shorter) forecast period self.assertEqual(result.coord("forecast_period").points, [4 * 3600]) # check time and frt points are also consistent with the UKV input cube self.assertEqual( result.coord("time").points, self.cube_ukv.coord("time").points) self.assertEqual( result.coord("forecast_reference_time").points, self.cube_ukv.coord("forecast_reference_time").points, ) def test_cycle_blend(self): """Test merge for blending over forecast_reference_time""" cube = self.cube_ukv.copy() cube.coord("forecast_reference_time").points = ( cube.coord("forecast_reference_time").points + 3600) cube.coord("forecast_period").points = ( cube.coord("forecast_reference_time").points - 3600) plugin = MergeCubesForWeightedBlending("forecast_reference_time") result = plugin.process([self.cube_ukv, cube]) self.assertIsInstance(result, iris.cube.Cube) self.assertIn(result.coord("forecast_reference_time"), result.coords(dim_coords=True)) # check no model coordinates have been added with self.assertRaises(iris.exceptions.CoordinateNotFoundError): result.coord(MODEL_BLEND_COORD) with self.assertRaises(iris.exceptions.CoordinateNotFoundError): result.coord(MODEL_NAME_COORD) def test_blend_coord_ascending(self): """Test the order of the output blend coordinate is always ascending, independent of the input cube order""" frt = self.cube_ukv.coord("forecast_reference_time").points[0] fp = self.cube_ukv.coord("forecast_period").points[0] cube1 = self.cube_ukv.copy() cube1.coord("forecast_reference_time").points = [frt + 3600] cube1.coord("forecast_period").points = [fp - 3600] cube2 = self.cube_ukv.copy() cube2.coord("forecast_reference_time").points = [frt + 7200] cube2.coord("forecast_period").points = [fp - 7200] # input unordered cubes; expect ordered output expected_points = np.array([frt, frt + 3600, frt + 7200], dtype=np.int64) plugin = MergeCubesForWeightedBlending("forecast_reference_time") result = plugin.process([cube1, self.cube_ukv, cube2]) self.assertArrayEqual( result.coord("forecast_reference_time").points, expected_points) def test_cycletime(self): """Test merged cube has updated forecast reference time and forecast period if specified using the 'cycletime' argument""" result = self.plugin.process(self.cubelist, cycletime="20151123T0600Z") # test resulting cube has forecast period consistent with cycletime self.assertEqual(result.coord("forecast_period").points, [3600]) self.assertEqual( result.coord("forecast_reference_time").points, self.cube_ukv.coord("forecast_reference_time").points + 3 * 3600, ) # check validity time is unchanged self.assertEqual( result.coord("time").points, self.cube_ukv.coord("time").points) def test_non_mo_model_id(self): """Test that a model ID attribute string can be specified when merging multi model cubes""" plugin = MergeCubesForWeightedBlending( "model", model_id_attr="non_mo_model_config") result = plugin.process(self.non_mo_cubelist) self.assertIsInstance(result, iris.cube.Cube) self.assertArrayEqual( result.coord(MODEL_BLEND_COORD).points, [0, 1000]) def test_model_id_attr_mismatch(self): """Test that when a model ID attribute string is specified that does not match the model ID attribute key name on both cubes to be merged, an error is thrown""" plugin = MergeCubesForWeightedBlending( "model", model_id_attr="non_matching_model_config") msg = "Cannot create model ID coordinate" with self.assertRaisesRegex(ValueError, msg): plugin.process(self.non_mo_cubelist) def test_model_id_attr_mismatch_one_cube(self): """Test that when a model ID attribute string is specified that only matches the model ID attribute key name on one of the cubes to be merged, an error is thrown""" self.non_mo_cubelist[1].attributes.pop("non_mo_model_config") self.non_mo_cubelist[1].attributes[ "non_matching_model_config"] = "non_uk_det" plugin = MergeCubesForWeightedBlending( "model", model_id_attr="non_matching_model_config") msg = "Cannot create model ID coordinate" with self.assertRaisesRegex(ValueError, msg): plugin.process(self.non_mo_cubelist) def test_time_bounds_mismatch(self): """Test failure for cycle blending when time bounds ranges are not matched (ie cycle blending different "accumulation periods")""" cube2 = self.cube_ukv.copy() cube2.coord("forecast_reference_time").points = ( cube2.coord("forecast_reference_time").points + 3600) cube2.coord("time").bounds = [ cube2.coord("time").bounds[0, 0] + 3600, cube2.coord("time").bounds[0, 1], ] cube2.coord("forecast_period").bounds = [ cube2.coord("forecast_period").bounds[0, 0] + 3600, cube2.coord("forecast_period").bounds[0, 1], ] msg = "Cube with mismatching time bounds ranges cannot be blended" with self.assertRaisesRegex(ValueError, msg): MergeCubesForWeightedBlending("forecast_reference_time").process( [self.cube_ukv, cube2]) def test_blend_coord_not_present(self): """Test exception when blend coord is not present on inputs""" msg = "realization coordinate is not present on all input cubes" with self.assertRaisesRegex(ValueError, msg): MergeCubesForWeightedBlending("realization").process(self.cubelist) def test_blend_realizations(self): """Test processing works for merging over coordinates that don't require specific setup""" data = np.ones((1, 3, 3), dtype=np.float32) cube1 = set_up_variable_cube(data, realizations=np.array([0])) cube1 = iris.util.squeeze(cube1) cube2 = set_up_variable_cube(data, realizations=np.array([1])) cube2 = iris.util.squeeze(cube2) plugin = MergeCubesForWeightedBlending("realization") result = plugin.process([cube1, cube2]) self.assertIsInstance(result, iris.cube.Cube) self.assertArrayEqual( result.coord("realization").points, np.array([0, 1])) self.assertEqual(result[0].metadata, cube1.metadata) self.assertEqual(result[1].metadata, cube2.metadata) def test_handling_blend_time(self): """Test merging works with mismatched and / or missing blend time coordinates""" blend_time_ukv = self.cube_ukv.coord("forecast_reference_time").copy() blend_time_ukv.rename("blend_time") self.cube_ukv.add_aux_coord(blend_time_ukv) blend_time_enuk = self.cube_enuk.coord( "forecast_reference_time").copy() blend_time_enuk.rename("blend_time") self.cube_enuk.add_aux_coord(blend_time_enuk) plugin = MergeCubesForWeightedBlending( "model_id", model_id_attr="mosg__model_configuration") result = plugin([self.cube_nowcast, self.cube_ukv, self.cube_enuk]) self.assertNotIn("blend_time", get_coord_names(result)) def test_forecast_coord_deprecation(self): """Test merging works if some (but not all) inputs have previously been cycle blended""" for cube in [self.cube_ukv, self.cube_enuk]: for coord in ["forecast_period", "forecast_reference_time"]: cube.coord(coord).attributes.update( {"deprecation_message": "blah"}) plugin = MergeCubesForWeightedBlending( "model_id", model_id_attr="mosg__model_configuration") result = plugin([self.cube_nowcast, self.cube_ukv, self.cube_enuk]) for coord in ["forecast_period", "forecast_reference_time"]: self.assertNotIn("deprecation_message", result.coord(coord).attributes)
def process(self, cubelist, cycletime=None, model_id_attr=None, spatial_weights=False, fuzzy_length=20000): """ Merge a cubelist, calculate appropriate blend weights and compute the weighted mean. Returns a single cube collapsed over the dimension given by self.blend_coord. Args: cubelist (iris.cube.CubeList): List of cubes to be merged and blended Kwargs: cycletime (str): Forecast reference time to use for output cubes, in the format YYYYMMDDTHHMMZ. If not set, the latest of the input cube forecast reference times is used. model_id_attr (str): Name of the attribute by which to identify the source model and construct "model" coordinates for blending. spatial_weights (bool): If true, calculate spatial weights. fuzzy_length (float): Distance (in metres) over which to smooth spatial weights. Default is 20 km. """ # Prepare cubes for weighted blending, including creating model_id and # model_configuration coordinates for multi-model blending. The merged # cube has a monotonically ascending blend coordinate. Plugin raises an # error if blend_coord is not present on all input cubes. merger = MergeCubesForWeightedBlending( self.blend_coord, weighting_coord=self.weighting_coord, model_id_attr=model_id_attr) cube = merger.process(cubelist, cycletime=cycletime) # if blend_coord has only one value, or is not present (case where only # one model has been provided for a model blend) update metadata only coord_names = [coord.name() for coord in cube.coords()] if (self.blend_coord not in coord_names or len(cube.coord(self.blend_coord).points) == 1): result = cube.copy() conform_metadata(result, cube, self.blend_coord, cycletime=cycletime) # otherwise, calculate weights and blend across specified dimension else: # set up special treatment for model blending if "model" in self.blend_coord: self.blend_coord = "model_id" # calculate blend weights weights = self._calculate_blending_weights(cube) if spatial_weights: weights = self._update_spatial_weights(cube, weights, fuzzy_length) # blend across specified dimension BlendingPlugin = WeightedBlendAcrossWholeDimension( self.blend_coord, cycletime=cycletime) result = BlendingPlugin.process(cube, weights=weights) return result
class Test_process(IrisTest): """Test the process method""" def setUp(self): """Set up some probability cubes from different models""" data = np.array([ 0.9 * np.ones((3, 3)), 0.5 * np.ones((3, 3)), 0.1 * np.ones((3, 3)) ], dtype=np.float32) thresholds = np.array([273., 275., 277.], dtype=np.float32) time_point = dt(2015, 11, 23, 7) time_bounds = [dt(2015, 11, 23, 4), time_point] # set up a MOGREPS-UK cube with 7 hour forecast period self.cube_enuk = set_up_probability_cube( data.copy(), thresholds, standard_grid_metadata='uk_ens', time=time_point, frt=dt(2015, 11, 23, 0), time_bounds=time_bounds) # set up a UKV cube with 4 hour forecast period self.cube_ukv = set_up_probability_cube( data.copy(), thresholds, standard_grid_metadata='uk_det', time=time_point, frt=dt(2015, 11, 23, 3), time_bounds=time_bounds) self.cubelist = iris.cube.CubeList([self.cube_enuk, self.cube_ukv]) # set up some non-UK test cubes cube_non_mo_ens = self.cube_enuk.copy() cube_non_mo_ens.attributes.pop("mosg__model_configuration") cube_non_mo_ens.attributes['non_mo_model_config'] = 'non_uk_ens' cube_non_mo_det = self.cube_ukv.copy() cube_non_mo_det.attributes.pop("mosg__model_configuration") cube_non_mo_det.attributes['non_mo_model_config'] = 'non_uk_det' self.non_mo_cubelist = iris.cube.CubeList( [cube_non_mo_ens, cube_non_mo_det]) # set up plugin for multi-model blending weighted by forecast period self.plugin = MergeCubesForWeightedBlending( "model", weighting_coord="forecast_period", model_id_attr="mosg__model_configuration") def test_basic(self): """Test single cube is returned unmodified""" cube = self.cube_enuk.copy() result = self.plugin.process(cube) self.assertArrayAlmostEqual(result.data, self.cube_enuk.data) self.assertEqual(result.metadata, self.cube_enuk.metadata) def test_single_item_list(self): """Test cube from single item list is returned unmodified""" cubelist = iris.cube.CubeList([self.cube_enuk.copy()]) result = self.plugin.process(cubelist) self.assertArrayAlmostEqual(result.data, self.cube_enuk.data) self.assertEqual(result.metadata, self.cube_enuk.metadata) def test_multi_model_merge(self): """Test models merge OK and have expected model coordinates""" result = self.plugin.process(self.cubelist) self.assertIsInstance(result, iris.cube.Cube) self.assertArrayEqual(result.coord("model_id").points, [0, 1000]) self.assertArrayEqual( result.coord("model_configuration").points, ["uk_ens", "uk_det"]) def test_time_coords(self): """Test merged cube has scalar time coordinates if weighting models by forecast period""" result = self.plugin.process(self.cubelist) # test resulting cube has single 4 hour (shorter) forecast period self.assertEqual(result.coord("forecast_period").points, [4 * 3600]) # check time and frt points are also consistent with the UKV input cube self.assertEqual( result.coord("time").points, self.cube_ukv.coord("time").points) self.assertEqual( result.coord("forecast_reference_time").points, self.cube_ukv.coord("forecast_reference_time").points) def test_cycle_blend(self): """Test merge for blending over forecast_reference_time""" cube = self.cube_ukv.copy() cube.coord("forecast_reference_time").points = ( cube.coord("forecast_reference_time").points + 3600) cube.coord("forecast_period").points = ( cube.coord("forecast_reference_time").points - 3600) plugin = MergeCubesForWeightedBlending("forecast_reference_time") result = plugin.process([self.cube_ukv, cube]) self.assertIsInstance(result, iris.cube.Cube) self.assertIn(result.coord("forecast_reference_time"), result.coords(dim_coords=True)) # check forecast period coordinate has been removed with self.assertRaises(iris.exceptions.CoordinateNotFoundError): result.coord("forecast_period") # check no model coordinates have been added with self.assertRaises(iris.exceptions.CoordinateNotFoundError): result.coord("model_id") with self.assertRaises(iris.exceptions.CoordinateNotFoundError): result.coord("model_configuration") def test_cycletime(self): """Test merged cube has updated forecast reference time and forecast period if specified using the 'cycletime' argument""" result = self.plugin.process(self.cubelist, cycletime="20151123T0600Z") # test resulting cube has forecast period consistent with cycletime self.assertEqual(result.coord("forecast_period").points, [3600]) self.assertEqual( result.coord("forecast_reference_time").points, self.cube_ukv.coord("forecast_reference_time").points + 3 * 3600) # check validity time is unchanged self.assertEqual( result.coord("time").points, self.cube_ukv.coord("time").points) def test_non_mo_model_id(self): """Test that a model ID attribute string can be specified when merging multi model cubes""" plugin = MergeCubesForWeightedBlending( "model", model_id_attr="non_mo_model_config") result = plugin.process(self.non_mo_cubelist) self.assertIsInstance(result, iris.cube.Cube) self.assertArrayEqual(result.coord("model_id").points, [0, 1000]) def test_model_id_attr_mismatch(self): """Test that when a model ID attribute string is specified that does not match the model ID attribute key name on both cubes to be merged, an error is thrown""" plugin = MergeCubesForWeightedBlending( "model", model_id_attr="non_matching_model_config") msg = "Cannot create model ID coordinate" with self.assertRaisesRegex(ValueError, msg): plugin.process(self.non_mo_cubelist) def test_model_id_attr_mismatch_one_cube(self): """Test that when a model ID attribute string is specified that only matches the model ID attribute key name on one of the cubes to be merged, an error is thrown""" self.non_mo_cubelist[1].attributes.pop("non_mo_model_config") self.non_mo_cubelist[1].attributes[ "non_matching_model_config"] = "non_uk_det" plugin = MergeCubesForWeightedBlending( "model", model_id_attr="non_matching_model_config") msg = "Cannot create model ID coordinate" with self.assertRaisesRegex(ValueError, msg): plugin.process(self.non_mo_cubelist) def test_time_bounds_mismatch(self): """Test failure for cycle blending when time bounds ranges are not matched (ie cycle blending different "accumulation periods")""" cube2 = self.cube_ukv.copy() cube2.coord("forecast_reference_time").points = ( cube2.coord("forecast_reference_time").points + 3600) cube2.coord("time").bounds = [ cube2.coord("time").bounds[0, 0] + 3600, cube2.coord("time").bounds[0, 1] ] cube2.coord("forecast_period").bounds = [ cube2.coord("forecast_period").bounds[0, 0] + 3600, cube2.coord("forecast_period").bounds[0, 1] ] msg = "Cube with mismatching time bounds ranges cannot be blended" with self.assertRaisesRegex(ValueError, msg): MergeCubesForWeightedBlending("forecast_reference_time").process( [self.cube_ukv, cube2]) def test_blend_coord_not_present(self): """Test exception when blend coord is not present on inputs""" msg = "realization coordinate is not present on all input cubes" with self.assertRaisesRegex(ValueError, msg): MergeCubesForWeightedBlending("realization").process(self.cubelist) def test_blend_realizations(self): """Test processing works for merging over coordinates that don't require specific setup""" data = np.ones((1, 3, 3), dtype=np.float32) cube1 = set_up_variable_cube(data, realizations=np.array([0])) cube1 = iris.util.squeeze(cube1) cube2 = set_up_variable_cube(data, realizations=np.array([1])) cube2 = iris.util.squeeze(cube2) plugin = MergeCubesForWeightedBlending("realization") result = plugin.process([cube1, cube2]) self.assertIsInstance(result, iris.cube.Cube) self.assertArrayEqual( result.coord("realization").points, np.array([0, 1])) self.assertEqual(result[0].metadata, cube1.metadata) self.assertEqual(result[1].metadata, cube2.metadata)