def _rationalise_blend_time_coords(self, cubelist, cycletime=None): """ Updates time coordinates on unmerged input cubes before blending depending on the coordinate over which the blend will be performed. Modifies cubes in place. If self.blend_coord is forecast_reference_time, ensures the cube does not have a forecast_period coordinate (this is recreated after blending). If self.weighting_coord is forecast_period, equalises forecast_reference_time on each cube before blending. Args: cubelist (iris.cube.CubeList): List of cubes containing data to be blended cycletime (str or None): The cycletime in a YYYYMMDDTHHMMZ format e.g. 20171122T0100Z Raises: ValueError: if forecast_reference_time (to be unified) is a dimension coordinate """ if "forecast_reference_time" in self.blend_coord: for cube in cubelist: coord_names = [x.name() for x in cube.coords()] if "forecast_period" in coord_names: cube.remove_coord("forecast_period") # if blending models using weights by forecast period, set forecast # reference times to current cycle time if ("model" in self.blend_coord and self.weighting_coord is not None and "forecast_period" in self.weighting_coord): cycletime = (find_latest_cycletime(cubelist) if cycletime is None else cycletime_to_datetime(cycletime)) unify_forecast_reference_time(cubelist, cycletime)
def test_define_cycletime_format(self): """Test when a cycletime is defined.""" cycletime = "201711220100" dt = datetime.datetime(2017, 11, 22, 1, 0) result = cycletime_to_datetime(cycletime, cycletime_format="%Y%m%d%H%M") self.assertEqual(result, dt)
def rebadge_forecasts_as_latest_cycle( cubes: Union[CubeList, List[Cube]], cycletime: Optional[str] = None ) -> CubeList: """ Function to update the forecast_reference_time and forecast_period on a list of input forecasts to match either a given cycletime, or the most recent forecast in the list (proxy for the current cycle). Args: cubes: Cubes that will have their forecast_reference_time and forecast_period updated. cycletime: Required forecast reference time in a YYYYMMDDTHHMMZ format e.g. 20171122T0100Z. If None, the latest forecast reference time is used. Returns: Updated cubes """ if cycletime is None and len(cubes) == 1: return cubes cycle_datetime = ( _find_latest_cycletime(cubes) if cycletime is None else cycletime_to_datetime(cycletime) ) return unify_cycletime(cubes, cycle_datetime)
def test_basic(self): """Test that a datetime object is returned of the expected value.""" cycletime = "20171122T0100Z" dt = datetime.datetime(2017, 11, 22, 1, 0) result = cycletime_to_datetime(cycletime) self.assertIsInstance(result, datetime.datetime) self.assertEqual(result, dt)
def add_blend_time(cube: Cube, cycletime: str) -> None: """ Function to add scalar blend time coordinate to a blended cube based on current cycle time. Modifies cube in place. Args: cubes: Cube to add blend time coordinate cycletime: Required blend time in a YYYYMMDDTHHMMZ format e.g. 20171122T0100Z """ cycle_datetime = cycletime_to_datetime(cycletime) blend_coord = _create_frt_type_coord(cube, cycle_datetime, name="blend_time") cube.add_aux_coord(blend_coord, data_dims=None)
def process(self, cubelist): """ Take an input cubelist containing forecasts from different cycles and merges them into a single cube. The steps taken are: 1. If no cycletime is given then find the latest cycle time from the input cubes. 2. Update the forecast periods in each input cube to be relative to the new cycletime. 3. Checks if there are duplicate realization numbers. If a duplicate is found, renumbers all of the realizations to remove any duplicates. 4. Merge cubes into one cube, removing any metadata that doesn't match. """ if self.cycletime is None: cycletime = find_latest_cycletime(cubelist) else: cycletime = cycletime_to_datetime(self.cycletime) cubelist = unify_forecast_reference_time(cubelist, cycletime) # Take all the realizations from all the input cube and # put in one array all_realizations = [ cube.coord("realization").points for cube in cubelist ] all_realizations = np.concatenate(all_realizations) # Find unique realizations unique_realizations = np.unique(all_realizations) # If we have fewer unique realizations than total realizations we have # duplicate realizations so we rebadge all realizations in the cubelist if len(unique_realizations) < len(all_realizations): first_realization = 0 for cube in cubelist: n_realization = len(cube.coord("realization").points) cube.coord("realization").points = np.arange( first_realization, first_realization + n_realization) first_realization = first_realization + n_realization # slice over realization to deal with cases where direct concatenation # would result in a non-monotonic coordinate lagged_ensemble = concatenate_cubes( cubelist, master_coord="realization", coords_to_slice_over=["realization"]) return lagged_ensemble
def rationalise_blend_time_coords(cubelist, blend_coord, cycletime=None, weighting_coord=None): """ Updates time coordinates on unmerged input cubes before blending depending on the coordinate over which the blend will be performed. Modifies cubes in place. If blend_coord is forecast_reference_time, ensures the cube does not have a forecast_period dimension. If weighting_coord is forecast_period, equalises forecast_reference_time on each cube before blending. Args: cubelist (iris.cube.CubeList): List of cubes containing data to be blended blend_coord (str): Name of coordinate over which the blend will be performed Kwargs: cycletime (str or None): The cycletime in a YYYYMMDDTHHMMZ format e.g. 20171122T0100Z weighting_coord (str or None): The coordinate across which weights will be scaled in a multi-model blend. Raises: ValueError: if forecast_reference_time (to be unified) is a dimension coordinate """ if "forecast_reference_time" in blend_coord: for cube in cubelist: coord_names = [x.name() for x in cube.coords()] if "forecast_period" in coord_names: cube.remove_coord("forecast_period") # if blending models using weights by forecast period, set forecast # reference times to current cycle time if ("model" in blend_coord and weighting_coord is not None and "forecast_period" in weighting_coord): if cycletime is None: cycletime = find_latest_cycletime(cubelist) else: cycletime = cycletime_to_datetime(cycletime) cubelist = unify_forecast_reference_time(cubelist, cycletime)
def process( start_cube: cli.inputcube, end_cube: cli.inputcube, *, interval_in_mins: int = None, times: cli.comma_separated_list = None, interpolation_method="linear", ): """Interpolate data between validity times. Interpolate data to intermediate times between the validity times of two cubes. This can be used to fill in missing data (e.g. for radar fields) or to ensure data is available at the required intervals when model data is not available at these times. Args: start_cube (iris.cube.Cube): Cube containing the data at the beginning. end_cube (iris.cube.Cube): Cube containing the data at the end. interval_in_mins (int): Specifies the interval in minutes at which to interpolate between the two input cubes. A number of minutes which does not divide up the interval equally will raise an exception. If intervals_in_mins is set then times can not be used. times (str): Specifies the times in the format {YYYYMMDD}T{HHMM}Z at which to interpolate between the two input cubes. Where {YYYYMMDD} is year, month, day and {HHMM} is hour and minutes e.g 20180116T0100Z. More than one time can be provided separated by a comma. If times are set, interval_in_mins can not be used. interpolation_method (str): ["linear", "solar", "daynight"] Specifies the interpolation method; solar interpolates using the solar elevation, daynight uses linear interpolation but sets night time points to 0.0 linear is linear interpolation. Returns: iris.cube.CubeList: A list of cubes interpolated to the desired times. The interpolated cubes will always be in chronological order of earliest to latest regardless of the order of the input. """ from improver.utilities.cube_manipulation import MergeCubes from improver.utilities.temporal import cycletime_to_datetime, iris_time_to_datetime from improver.utilities.temporal_interpolation import TemporalInterpolation (time_start,) = iris_time_to_datetime(start_cube.coord("time")) (time_end,) = iris_time_to_datetime(end_cube.coord("time")) if time_end < time_start: # swap cubes start_cube, end_cube = end_cube, start_cube if times is not None: times = [cycletime_to_datetime(timestr) for timestr in times] result = TemporalInterpolation( interval_in_minutes=interval_in_mins, times=times, interpolation_method=interpolation_method, )(start_cube, end_cube) return MergeCubes()(result)
def process( mandatory_attributes_json: cli.inputjson, *, name="air_pressure_at_sea_level", units=None, spatial_grid="latlon", time_period: int = None, json_input: cli.inputjson = None, ensemble_members: int = 8, grid_spacing: float = None, domain_corner: cli.comma_separated_list_of_float = None, npoints: int = 71, ): """ Generate a cube with metadata only. Args: mandatory_attributes_json (Dict): Specifies the values of the mandatory attributes, title, institution and source. name (Optional[str]): Output variable name, or if creating a probability cube the name of the underlying variable to which the probability field applies. units (Optional[str]): Output variable units, or if creating a probability cube the units of the underlying variable / threshold. spatial_grid (Optional[str]): What type of x/y coordinate values to use. Permitted values are "latlon" or "equalarea". time_period (Optional[int]): The period in minutes between the time bounds. This is used to calculate the lower time bound. If unset the diagnostic will be instantaneous, i.e. without time bounds. json_input (Optional[Dict]): Dictionary containing values for one or more of: "name", "units", "time", "time_bounds", "frt", "spp__relative_to_threshold", "attributes" (dictionary of additional metadata attributes) and "coords" (dictionary). "coords" can contain "height_levels" (list of height/pressure level values), and one of "realizations", "percentiles" or "thresholds" (list of dimension values). ensemble_members (Optional[int]): Number of ensemble members. Default 8. Will not be used if "realizations", "percentiles" or "thresholds" provided in json_input. grid_spacing (Optional[float]): Resolution of grid (metres or degrees). domain_corner (Optional[Tuple[float, float]]): Bottom left corner of grid domain (y,x) (degrees for latlon or metres for equalarea). npoints (Optional[int]): Number of points along each of the y and x spatial axes. Returns: iris.cube.Cube: Output of generate_metadata() """ # Set arguments to pass to generate_metadata function and remove json_input for # processing contents before adding generate_metadata_args = locals() for key in ["mandatory_attributes_json", "json_input"]: generate_metadata_args.pop(key, None) from improver.synthetic_data.generate_metadata import generate_metadata from improver.synthetic_data.utilities import ( get_height_levels, get_leading_dimension, ) from improver.utilities.temporal import cycletime_to_datetime if json_input is not None: # Get leading dimension and height/pressure data from json_input if "coords" in json_input: coord_data = json_input["coords"] ( json_input["leading_dimension"], json_input["cube_type"], ) = get_leading_dimension(coord_data) json_input["height_levels"], json_input["pressure"] = get_height_levels( coord_data ) json_input.pop("coords", None) # Convert str time, frt and time_bounds to datetime if "time" in json_input: json_input["time"] = cycletime_to_datetime(json_input["time"]) if "frt" in json_input: json_input["frt"] = cycletime_to_datetime(json_input["frt"]) if "time_bounds" in json_input: time_bounds = [] for tb in json_input["time_bounds"]: time_bounds.append(cycletime_to_datetime(tb)) json_input["time_bounds"] = time_bounds # Update generate_metadata_args with the json_input data generate_metadata_args.update(json_input) return generate_metadata(mandatory_attributes_json, **generate_metadata_args)
def process(cube_0, cube_1, interval_in_mins=None, in_times=None, interpolation_method='linear'): """Module to interpolate data between validity times. Interpolate data to intermediate times between the validity times of two cubes. This can be used to fill in missing data (e.g. for radar fields) or to ensure data is available at the required intervals when model data is not available at these times. Args: cube_0 (iris.cube.Cube): Cube containing the data at the beginning. cube_1 (iris.cube.Cube): Cube containing the data at the end. interval_in_mins (int): Specifies the interval in minutes at which to interpolate between the two input cubes. A number of minutes which does not divide up the interval equally will raise an exception. If intervals_in_mins is set then in_times can not be used. Default is None. in_times (str): Specifies the times in the format {YYYYMMDD}T{HHMM}Z at which to interpolate between the two input cubes. Where {YYYYMMDD} is year, month, day and {HHMM} is hour and minutes e.g 20180116T0100Z. More than one time can be provided separated by a space. If in_times are set, interval_in_mins can not be used. Default is None. interpolation_method (str): ["linear", "solar", "daynight"] Specifies the interpolation method; solar interpolates using the solar elevation, daynight uses linear interpolation but sets night time points to 0.0 linear is linear interpolation. Default is linear. Returns: iris.cube.CubeList: A list of cubes interpolated to the desired times. The interpolated cubes will always be in chronological order of earliest to latest regardless of the order of the input. """ time_0, = iris_time_to_datetime(cube_0.coord('time')) time_1, = iris_time_to_datetime(cube_1.coord('time')) if time_0 < time_1: cube_start = cube_0 cube_end = cube_1 else: cube_start = cube_1 cube_end = cube_0 interval = interval_in_mins method = interpolation_method if in_times is None: times = in_times else: times = [] for timestr in in_times: times.append(cycletime_to_datetime(timestr)) result = (TemporalInterpolation(interval_in_minutes=interval, times=times, interpolation_method=method).process( cube_start, cube_end)) return result
def create_coefficients_cube( self, optimised_coeffs, historic_forecast): """Create a cube for storing the coefficients computed using EMOS. .. See the documentation for examples of these cubes. .. include:: extended_documentation/ensemble_calibration/ ensemble_calibration/create_coefficients_cube.rst Args: optimised_coeffs (list): List of optimised coefficients. Order of coefficients is [gamma, delta, alpha, beta]. historic_forecast (iris.cube.Cube): The cube containing the historic forecast. Returns: cube (iris.cube.Cube): Cube constructed using the coefficients provided and using metadata from the historic_forecast cube. The cube contains a coefficient_index dimension coordinate where the points of the coordinate are integer values and a coefficient_name auxiliary coordinate where the points of the coordinate are e.g. gamma, delta, alpha, beta. """ if self.predictor_of_mean_flag.lower() == "realizations": realization_coeffs = [] for realization in historic_forecast.coord("realization").points: realization_coeffs.append( "{}{}".format(self.coeff_names[-1], np.int32(realization))) coeff_names = self.coeff_names[:-1] + realization_coeffs else: coeff_names = self.coeff_names if len(optimised_coeffs) != len(coeff_names): msg = ("The number of coefficients in {} must equal the " "number of coefficient names {}.".format( optimised_coeffs, coeff_names)) raise ValueError(msg) coefficient_index = iris.coords.DimCoord( np.arange(len(optimised_coeffs), dtype=np.int32), long_name="coefficient_index", units="1") coefficient_name = iris.coords.AuxCoord( coeff_names, long_name="coefficient_name", units="no_unit") dim_coords_and_dims = [(coefficient_index, 0)] aux_coords_and_dims = [(coefficient_name, 0)] # Create a forecast_reference_time coordinate. frt_point = cycletime_to_datetime(self.current_cycle) try: frt_coord = ( historic_forecast.coord("forecast_reference_time").copy( datetime_to_iris_time(frt_point, time_units="seconds"))) except CoordinateNotFoundError: pass else: aux_coords_and_dims.append((frt_coord, None)) # Create forecast period and time coordinates. try: fp_point = ( np.unique(historic_forecast.coord("forecast_period").points)) fp_coord = ( historic_forecast.coord("forecast_period").copy(fp_point)) except CoordinateNotFoundError: pass else: aux_coords_and_dims.append((fp_coord, None)) if historic_forecast.coords("time"): frt_point = cycletime_to_datetime(self.current_cycle) # Ensure that the fp_point is determined with units of seconds. copy_of_fp_coord = ( historic_forecast.coord("forecast_period").copy()) copy_of_fp_coord.convert_units("seconds") fp_point, = np.unique(copy_of_fp_coord.points) time_point = ( frt_point + datetime.timedelta(seconds=float(fp_point))) time_point = datetime_to_iris_time( time_point, time_units=str(historic_forecast.coord("time").units)) time_coord = historic_forecast.coord("time").copy(time_point) aux_coords_and_dims.append((time_coord, None)) attributes = {"diagnostic_standard_name": historic_forecast.name()} for attribute in historic_forecast.attributes.keys(): if attribute.endswith("model_configuration"): attributes[attribute] = ( historic_forecast.attributes[attribute]) cube = iris.cube.Cube( optimised_coeffs, long_name="emos_coefficients", units="1", dim_coords_and_dims=dim_coords_and_dims, aux_coords_and_dims=aux_coords_and_dims, attributes=attributes) return cube
def main(argv=None): """ Interpolate data to intermediate times between the validity times of two cubes. This can be used to fill in missing data (e.g. for radar fields) or to ensure data is available at the required intervals when model data is not available at these times. """ parser = ArgParser(description='Interpolate data between validity times ') parser.add_argument('infiles', metavar='INFILES', nargs=2, help='Files contain the data at the beginning' ' and end of the period (2 files required).') group = parser.add_mutually_exclusive_group(required=True) group.add_argument("--interval_in_mins", metavar="INTERVAL_IN_MINS", default=None, type=int, help="Specifies the interval in minutes" " at which to interpolate " "between the two input cubes." " A number of minutes which does not " "divide up the interval equally will " "raise an exception. If intervals_in_mins " "is set then times can not be set.") group.add_argument("--times", metavar="TIMES", default=None, nargs="+", type=str, help="Specifies the times in the format " "{YYYYMMDD}T{HHMM}Z " " at which to interpolate " "between the two input cubes." "Where {YYYYMMDD} is year, month day " "and {HHMM} is hour and minutes e.g " "20180116T0100Z. More than one time" "can be provided separated by a space " "but if times are set interval_in_mins " "can not be set") parser.add_argument("--interpolation_method", metavar="INTERPOLATION_METHOD", default="linear", choices=["linear", "solar", "daynight"], help="Specifies the interpolation method; " "solar interpolates using the solar elevation, " "daynight uses linear interpolation but sets" " night time points to 0.0, " "linear is linear interpolation. " "Default is linear.") parser.add_argument("--output_files", metavar="OUTPUT_FILES", required=True, nargs="+", help="List of output files." " The interpolated files will always be" " in the chronological order of" " earliest to latest " " regardless of the order of the infiles.") args = parser.parse_args(args=argv) cube_0 = load_cube(args.infiles[0]) cube_1 = load_cube(args.infiles[1]) time_0, = iris_time_to_datetime(cube_0.coord('time')) time_1, = iris_time_to_datetime(cube_1.coord('time')) if time_0 < time_1: cube_start = cube_0 cube_end = cube_1 else: cube_start = cube_1 cube_end = cube_0 interval = args.interval_in_mins method = args.interpolation_method if args.times is None: times = args.times else: times = [] for timestr in args.times: times.append(cycletime_to_datetime(timestr)) interpolated_cubes = (TemporalInterpolation( interval_in_minutes=interval, times=times, interpolation_method=method).process(cube_start, cube_end)) len_files = len(args.output_files) len_cubes = len(interpolated_cubes) if len_files == len_cubes: for i, cube_out in enumerate(interpolated_cubes): save_netcdf(cube_out, args.output_files[i]) else: msg = ("Output_files do not match cubes created. " "{} files given but {} required.".format(len_files, len_cubes)) raise ValueError(msg)