def test_wildcard_files_with_constraint(self):
"""Test that the loading works correctly, if a wildcarded filepath is
provided and a constraint is provided that is only valid for a subset
of the available files."""
low_cloud_cube = self.cube.copy()
low_cloud_cube.rename("low_type_cloud_area_fraction")
low_cloud_cube.units = 1
save_netcdf(low_cloud_cube, self.low_cloud_filepath)
medium_cloud_cube = self.cube.copy()
medium_cloud_cube.rename("medium_type_cloud_area_fraction")
medium_cloud_cube.units = 1
save_netcdf(medium_cloud_cube, self.med_cloud_filepath)
constr = iris.Constraint("low_type_cloud_area_fraction")
result = load_cubelist(
[self.low_cloud_filepath, self.med_cloud_filepath],
constraints=constr)
self.assertEqual(len(result), 1)
self.assertArrayAlmostEqual(result[0].coord("realization").points,
self.realization_points)
self.assertArrayAlmostEqual(result[0].coord("time").points,
self.time_points)
self.assertArrayAlmostEqual(result[0].coord("latitude").points,
self.latitude_points)
self.assertArrayAlmostEqual(result[0].coord("longitude").points,
self.longitude_points)
def main(argv=None):
"""Parser to accept input data and an output destination before invoking
the weather symbols plugin.
"""
diagnostics = interrogate_decision_tree('high_resolution')
n_files = len(diagnostics)
dlist = (' - {}\n' * n_files)
diagnostics_global = interrogate_decision_tree('global')
n_files_global = len(diagnostics_global)
dlist_global = (' - {}\n' * n_files_global)
parser = ArgParser(
description='Calculate gridded weather symbol codes.\nThis plugin '
'requires a specific set of input diagnostics, where data\nmay be in '
'any units to which the thresholds given below can\nbe converted:\n' +
dlist.format(*diagnostics) + '\n\n or for global data\n\n' +
dlist_global.format(*diagnostics_global),
formatter_class=RawTextHelpFormatter)
parser.add_argument(
'input_filepaths',
metavar='INPUT_FILES',
nargs="+",
help='Paths to files containing the required input diagnostics.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
parser.add_argument("--wxtree",
metavar="WXTREE",
default="high_resolution",
choices=["high_resolution", "global"],
help="Weather Code tree.\n"
"Choices are high_resolution or global.\n"
"Default=high_resolution.",
type=str)
args = parser.parse_args(args=argv)
# Load Cube
cubes = load_cubelist(args.input_filepaths, no_lazy_load=True)
required_number_of_inputs = n_files
if args.wxtree == 'global':
required_number_of_inputs = n_files_global
if len(cubes) != required_number_of_inputs:
msg = ('Incorrect number of inputs: files {} gave {} cubes' +
', {} required').format(args.input_filepaths, len(cubes),
required_number_of_inputs)
raise argparse.ArgumentTypeError(msg)
# Process Cube
result = process(cubes, args.wxtree)
# Save Cube
save_netcdf(result, args.output_filepath)
def test_single_file(self):
"""Test that the loading works correctly, if only the filepath is
provided."""
result = load_cubelist(self.filepath)
self.assertArrayAlmostEqual(result[0].coord("realization").points,
self.realization_points)
self.assertArrayAlmostEqual(result[0].coord("time").points,
self.time_points)
self.assertArrayAlmostEqual(result[0].coord("latitude").points,
self.latitude_points)
self.assertArrayAlmostEqual(result[0].coord("longitude").points,
self.longitude_points)
def test_wildcard_files(self):
"""Test that the loading works correctly, if a wildcarded filepath is
provided."""
filepath = os.path.join(self.directory, "*.nc")
result = load_cubelist(filepath)
self.assertArrayAlmostEqual(result[0].coord("realization").points,
self.realization_points)
self.assertArrayAlmostEqual(result[0].coord("time").points,
self.time_points)
self.assertArrayAlmostEqual(result[0].coord("latitude").points,
self.latitude_points)
self.assertArrayAlmostEqual(result[0].coord("longitude").points,
self.longitude_points)
def test_multiple_files(self):
"""Test that the loading works correctly, if a path to multiple files
is provided."""
result = load_cubelist([self.filepath, self.filepath])
for cube in result:
self.assertArrayAlmostEqual(
cube.coord("realization").points, self.realization_points)
self.assertArrayAlmostEqual(
cube.coord("time").points, self.time_points)
self.assertArrayAlmostEqual(
cube.coord("latitude").points, self.latitude_points)
self.assertArrayAlmostEqual(
cube.coord("longitude").points, self.longitude_points)
def test_no_lazy_load(self):
"""Test that the loading works correctly with lazy load bypassing."""
result = load_cubelist([self.filepath, self.filepath],
no_lazy_load=True)
self.assertIsInstance(result, iris.cube.CubeList)
self.assertArrayEqual([False, False],
[_.has_lazy_data() for _ in result])
for cube in result:
self.assertArrayAlmostEqual(
cube.coord("realization").points, self.realization_points)
self.assertArrayAlmostEqual(
cube.coord("time").points, self.time_points)
self.assertArrayAlmostEqual(
cube.coord("latitude").points, self.latitude_points)
self.assertArrayAlmostEqual(
cube.coord("longitude").points, self.longitude_points)
def test_no_lazy_load(self):
"""Test that the cubelist returned upon loading does not contain
lazy data."""
result = load_cubelist([self.filepath, self.filepath],
no_lazy_load=True)
self.assertArrayEqual([False, False],
[_.has_lazy_data() for _ in result])
for cube in result:
self.assertArrayAlmostEqual(
cube.coord("realization").points, self.realization_points)
self.assertArrayAlmostEqual(
cube.coord("time").points, self.time_points)
self.assertArrayAlmostEqual(
cube.coord("latitude").points, self.latitude_points)
self.assertArrayAlmostEqual(
cube.coord("longitude").points, self.longitude_points)
def test_no_partial_merge_single_arg(self):
"""Test that we can load three files independently when a wildcarded
filepath is provided, even if two of the cubes could be merged"""
low_cloud_cube = self.cube.copy()
low_cloud_cube.rename("low_type_cloud_area_fraction")
low_cloud_cube.coord("time").points = (
low_cloud_cube.coord("time").points + 3600)
low_cloud_cube.coord("forecast_period").points = (
low_cloud_cube.coord("forecast_period").points - 3600)
save_netcdf(low_cloud_cube, self.low_cloud_filepath)
medium_cloud_cube = self.cube.copy()
medium_cloud_cube.rename("medium_type_cloud_area_fraction")
save_netcdf(medium_cloud_cube, self.med_cloud_filepath)
fileglob = os.path.join(self.directory, "*.nc")
result = load_cubelist(fileglob)
self.assertEqual(len(result), 3)
def get_additional_diagnostics(diagnostic_name,
diagnostic_data_path,
time_extract=None):
"""
Load additional diagnostics needed for particular spot data processes.
Args:
diagnostic_name (string):
The name of the diagnostic to be loaded. Used to find
the relevant file.
time_extract (iris.Constraint):
An iris constraint to extract and return only data from the desired
time.
Returns:
cube (iris.cube.CubeList):
CubeList containing the desired diagnostic data, with a single
entry if time_extract is provided.
Raises:
IOError : If files are not found.
ValueError : If required diagnostics are not found in the read files.
"""
# Search diadnostic data directory for all files relevant to current
# diagnostic.
files_to_read = [
os.path.join(dirpath, filename)
for dirpath, _, files in os.walk(diagnostic_data_path)
for filename in files if diagnostic_name in filename
]
if not files_to_read:
raise IOError('The relevant data files for {}, which is required '
'as an additional diagnostic is not available '
'in {}.'.format(diagnostic_name, diagnostic_data_path))
cubes = load_cubelist(files_to_read)
if time_extract is not None:
with iris.FUTURE.context(cell_datetime_objects=True):
cubes = cubes.extract(time_extract)
if not cubes:
raise ValueError('No diagnostics match {}'.format(time_extract))
return cubes
def test_no_partial_merge_list_args(self):
"""Test that we can load three files independently when a wildcarded
filepath is provided in a single-item list. This is the form in which
multi-item arguments ("nargs=+") are provided via "argparse" from the
loading CLIs."""
low_cloud_cube = self.cube.copy()
low_cloud_cube.rename("low_type_cloud_area_fraction")
low_cloud_cube.coord("time").points = (
low_cloud_cube.coord("time").points + 3600)
low_cloud_cube.coord("forecast_period").points = (
low_cloud_cube.coord("forecast_period").points - 3600)
save_netcdf(low_cloud_cube, self.low_cloud_filepath)
medium_cloud_cube = self.cube.copy()
medium_cloud_cube.rename("medium_type_cloud_area_fraction")
save_netcdf(medium_cloud_cube, self.med_cloud_filepath)
fileglob = os.path.join(self.directory, "*.nc")
result = load_cubelist([fileglob])
self.assertEqual(len(result), 3)
def main(argv=None):
"""Load in arguments and ensure they are set correctly.
Then run Triangular weighted blending across the given coordinate."""
parser = ArgParser(
description='Use the TriangularWeightedBlendAcrossAdjacentPoints to '
'blend across a particular coordinate. It does not '
'collapse the coordinate, but instead blends across '
'adjacent points and puts the blended values back in the '
'original coordinate, with adjusted bounds.')
parser.add_argument('coordinate', type=str,
metavar='COORDINATE_TO_BLEND_OVER',
help='The coordinate over which the blending '
'will be applied.')
parser.add_argument('central_point', metavar='CENTRAL_POINT', type=float,
help='Central point at which the output from the '
'triangular weighted blending will be '
'calculated. This should be in the units of the '
'units argument that is passed in. '
'This value should be a point on the '
'coordinate for blending over.')
parser.add_argument('--units', metavar='UNIT_STRING', required=True,
help='Units of the the central_point and width.')
parser.add_argument('--calendar', metavar='CALENDAR',
default='gregorian',
help='Calendar for parameter_unit if required. '
'Default=gregorian')
parser.add_argument('--width', metavar='TRIANGLE_WIDTH', type=float,
required=True,
help='Width of the triangular weighting function used '
'in the blending, in the units of the '
'units argument passed in.')
parser.add_argument('--blend_time_using_forecast_period',
default=False, action='store_true', help='Flag that '
'we are blending over time but using the forecast '
'period coordinate as a proxy. Note this should only '
'be used when time and forecast_period share a '
'dimension: ie when all files provided are from the '
'same forecast cycle.')
parser.add_argument('input_filepaths', metavar='INPUT_FILES', nargs="+",
help='Paths to input NetCDF files including and '
'surrounding the central_point.')
parser.add_argument('output_filepath', metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
args = parser.parse_args(args=argv)
# TriangularWeightedBlendAcrossAdjacentPoints can't currently handle
# blending over times where iris reads the coordinate points as datetime
# objects. Fail here to avoid unhelpful errors downstream.
if "time" in args.coordinate:
msg = ("Cannot blend over {} coordinate (points encoded as datetime "
"objects)".format(args.coordinate))
raise ValueError(msg)
# This is left as a placeholder for when we have this capability
if args.coordinate == 'time':
units = Unit(args.units, args.calendar)
else:
units = args.units
cubelist = load_cubelist(args.input_filepaths)
if (args.blend_time_using_forecast_period and
args.coordinate == 'forecast_period'):
cube = MergeCubes().process(cubelist, check_time_bounds_ranges=True)
elif args.blend_time_using_forecast_period:
msg = ('"--blend_time_using_forecast_period" can only be used with '
'"forecast_period" coordinate')
raise ValueError(msg)
else:
cube = MergeCubes().process(cubelist)
BlendingPlugin = TriangularWeightedBlendAcrossAdjacentPoints(
args.coordinate, args.central_point, units, args.width)
result = BlendingPlugin.process(cube)
save_netcdf(result, args.output_filepath)
def test_lazy_load(self):
"""Test that the cubelist returned upon loading does contain
lazy data."""
result = load_cubelist([self.filepath, self.filepath])
self.assertArrayEqual([True, True],
[_.has_lazy_data() for _ in result])
def main(argv=None):
"""Calculate optical flow advection velocities"""
parser = ArgParser(
description="Calculate optical flow components from input fields.")
parser.add_argument("input_filepaths",
metavar="INPUT_FILEPATHS",
nargs=3,
type=str,
help="Paths to the input radar "
"files. There should be 3 input files at T, T-1 and "
"T-2 from which to calculate optical flow velocities. "
"The files require a 'time' coordinate on which they "
"are sorted, so the order of inputs does not matter.")
parser.add_argument("output_filepath",
metavar="OUTPUT_FILEPATH",
help="The output path for the resulting NetCDF")
parser.add_argument("--nowcast_filepaths",
nargs="+",
type=str,
default=None,
help="Optional list of full paths to "
"output nowcast files. Overrides OUTPUT_DIR. Ignored "
"unless '--extrapolate' is set.")
parser.add_argument("--orographic_enhancement_filepaths",
nargs="+",
type=str,
default=None,
help="List or wildcarded "
"file specification to the input orographic "
"enhancement files. Orographic enhancement files are "
"compulsory for precipitation fields.")
parser.add_argument("--json_file",
metavar="JSON_FILE",
default=None,
help="Filename for the json file containing "
"required changes to attributes. "
"Every output file will have the attributes_dict "
"applied. Defaults to None.",
type=str)
# OpticalFlow plugin configurable parameters
parser.add_argument("--ofc_box_size",
type=int,
default=30,
help="Size of "
"square 'box' (in grid squares) within which to solve "
"the optical flow equations.")
parser.add_argument("--smart_smoothing_iterations",
type=int,
default=100,
help="Number of iterations to perform in enforcing "
"smoothness constraint for optical flow velocities.")
args = parser.parse_args(args=argv)
# Load Cubes and JSON
attributes_dict = load_json_or_none(args.json_file)
original_cube_list = load_cubelist(args.input_filepaths)
oe_cube = load_cube(args.orographic_enhancement_filepaths, allow_none=True)
# Process
result = process(original_cube_list, oe_cube, attributes_dict,
args.ofc_box_size, args.smart_smoothing_iterations)
# Save Cubes
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Load in arguments and ensure they are set correctly.
Then run Triangular weighted blending across the given coordinate."""
parser = ArgParser(
description='Use the TriangularWeightedBlendAcrossAdjacentPoints to '
'blend across a particular coordinate. It does not '
'collapse the coordinate, but instead blends across '
'adjacent points and puts the blended values back in the '
'original coordinate, with adjusted bounds.')
parser.add_argument('coordinate',
type=str,
metavar='COORDINATE_TO_BLEND_OVER',
help='The coordinate over which the blending '
'will be applied.')
parser.add_argument('central_point',
metavar='CENTRAL_POINT',
type=float,
help='Central point at which the output from the '
'triangular weighted blending will be '
'calculated. This should be in the units of the '
'units argument that is passed in. '
'This value should be a point on the '
'coordinate for blending over.')
parser.add_argument('--units',
metavar='UNIT_STRING',
required=True,
help='Units of the central_point and width.')
parser.add_argument('--calendar',
metavar='CALENDAR',
default='gregorian',
help='Calendar for parameter_unit if required. '
'Default=gregorian')
parser.add_argument('--width',
metavar='TRIANGLE_WIDTH',
type=float,
required=True,
help='Width of the triangular weighting function used '
'in the blending, in the units of the '
'units argument passed in.')
parser.add_argument('--blend_time_using_forecast_period',
default=False,
action='store_true',
help='Flag that '
'we are blending over time but using the forecast '
'period coordinate as a proxy. Note this should only '
'be used when time and forecast_period share a '
'dimension: ie when all files provided are from the '
'same forecast cycle.')
parser.add_argument('input_filepaths',
metavar='INPUT_FILES',
nargs="+",
help='Paths to input NetCDF files including and '
'surrounding the central_point.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
args = parser.parse_args(args=argv)
# Load Cubelist
cubelist = load_cubelist(args.input_filepaths)
# Process Cube
result = process(cubelist, args.coordinate, args.central_point, args.units,
args.width, args.calendar,
args.blend_time_using_forecast_period)
# Save Cube
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Load in arguments and ensure they are set correctly.
Then load in the data to blend and calculate default weights
using the method chosen before carrying out the blending."""
parser = ArgParser(
description='Calculate the default weights to apply in weighted '
'blending plugins using the ChooseDefaultWeightsLinear or '
'ChooseDefaultWeightsNonLinear plugins. Then apply these '
'weights to the dataset using the BasicWeightedAverage plugin.'
' Required for ChooseDefaultWeightsLinear: y0val and ynval.'
' Required for ChooseDefaultWeightsNonLinear: cval.'
' Required for ChooseWeightsLinear with dict: wts_dict.')
parser.add_argument('--wts_calc_method',
metavar='WEIGHTS_CALCULATION_METHOD',
choices=['linear', 'nonlinear', 'dict'],
default='linear',
help='Method to use to calculate '
'weights used in blending. "linear" (default): '
'calculate linearly varying blending weights. '
'"nonlinear": calculate blending weights that decrease'
' exponentially with increasing blending coordinate. '
'"dict": calculate weights using a dictionary passed '
'in as a command line argument.')
parser.add_argument('coordinate',
type=str,
metavar='COORDINATE_TO_AVERAGE_OVER',
help='The coordinate over which the blending '
'will be applied.')
parser.add_argument('--cycletime',
metavar='CYCLETIME',
type=str,
help='The forecast reference time to be used after '
'blending has been applied, in the format '
'YYYYMMDDTHHMMZ. If not provided, the blended file '
'will take the latest available forecast reference '
'time from the input cubes supplied.')
parser.add_argument('--model_id_attr',
metavar='MODEL_ID_ATTR',
type=str,
default=None,
help='The name of the netCDF file attribute to be '
'used to identify the source model for '
'multi-model blends. Default is None. '
'Must be present on all input '
'files if blending over models.')
parser.add_argument('--spatial_weights_from_mask',
action='store_true',
default=False,
help='If set this option will result in the generation'
' of spatially varying weights based on the'
' masks of the data we are blending. The'
' one dimensional weights are first calculated '
' using the chosen weights calculation method,'
' but the weights will then be adjusted spatially'
' based on where there is masked data in the data'
' we are blending. The spatial weights are'
' calculated using the'
' SpatiallyVaryingWeightsFromMask plugin.')
parser.add_argument('input_filepaths',
metavar='INPUT_FILES',
nargs="+",
help='Paths to input files to be blended.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
spatial = parser.add_argument_group(
'Spatial weights from mask options',
'Options for calculating the spatial weights using the '
'SpatiallyVaryingWeightsFromMask plugin.')
spatial.add_argument('--fuzzy_length',
metavar='FUZZY_LENGTH',
type=float,
default=20000,
help='When calculating spatially varying weights we'
' can smooth the weights so that areas close to'
' areas that are masked have lower weights than'
' those further away. This fuzzy length controls'
' the scale over which the weights are smoothed.'
' The fuzzy length is in terms of m, the'
' default is 20km. This distance is then'
' converted into a number of grid squares,'
' which does not have to be an integer. Assumes'
' the grid spacing is the same in the x and y'
' directions, and raises an error if this is not'
' true. See SpatiallyVaryingWeightsFromMask for'
' more detail.')
linear = parser.add_argument_group(
'linear weights options', 'Options for the linear weights '
'calculation in '
'ChooseDefaultWeightsLinear')
linear.add_argument('--y0val',
metavar='LINEAR_STARTING_POINT',
type=float,
help='The relative value of the weighting start point '
'(lowest value of blend coord) for choosing default '
'linear weights. This must be a positive float or 0.')
linear.add_argument('--ynval',
metavar='LINEAR_END_POINT',
type=float,
help='The relative value of the weighting '
'end point (highest value of blend coord) for choosing'
' default linear weights. This must be a positive '
'float or 0. Note that if blending over forecast '
'reference time, ynval >= y0val would normally be '
'expected (to give greater weight to the more recent '
'forecast).')
nonlinear = parser.add_argument_group(
'nonlinear weights options', 'Options for the non-linear '
'weights calculation in '
'ChooseDefaultWeightsNonLinear')
nonlinear.add_argument('--cval',
metavar='NON_LINEAR_FACTOR',
type=float,
help='Factor used to determine how skewed the '
'non linear weights will be. A value of 1 '
'implies equal weighting.')
wts_dict = parser.add_argument_group(
'dict weights options', 'Options for linear weights to be '
'calculated based on parameters '
'read from a json file dict')
wts_dict.add_argument('--wts_dict',
metavar='WEIGHTS_DICTIONARY',
help='Path to json file containing dictionary from '
'which to calculate blending weights. Dictionary '
'format is as specified in the improver.blending.'
'weights.ChooseWeightsLinear plugin.')
wts_dict.add_argument('--weighting_coord',
metavar='WEIGHTING_COORD',
default='forecast_period',
help='Name of '
'coordinate over which linear weights should be '
'scaled. This coordinate must be available in the '
'weights dictionary.')
args = parser.parse_args(args=argv)
# reject incorrect argument combinations
if (args.wts_calc_method == "linear") and args.cval:
parser.wrong_args_error('cval', 'linear')
if ((args.wts_calc_method == "nonlinear")
and np.any([args.y0val, args.ynval])):
parser.wrong_args_error('y0val, ynval', 'non-linear')
if (args.wts_calc_method == "dict") and not args.wts_dict:
parser.error('Dictionary is required if --wts_calc_method="dict"')
weights_dict = load_json_or_none(args.wts_dict)
# Load cubes to be blended.
cubelist = load_cubelist(args.input_filepaths)
result = process(cubelist, args.wts_calc_method, args.coordinate,
args.cycletime, args.weighting_coord, weights_dict,
args.y0val, args.ynval, args.cval, args.model_id_attr,
args.spatial_weights_from_mask, args.fuzzy_length)
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Calculate optical flow advection velocities and (optionally)
extrapolate data."""
parser = ArgParser(
description="Calculate optical flow components from input fields "
"and (optionally) extrapolate to required lead times.")
parser.add_argument("input_filepaths",
metavar="INPUT_FILEPATHS",
nargs=3,
type=str,
help="Paths to the input radar "
"files. There should be 3 input files at T, T-1 and "
"T-2 from which to calculate optical flow velocities. "
"The files require a 'time' coordinate on which they "
"are sorted, so the order of inputs does not matter.")
parser.add_argument("--output_dir",
metavar="OUTPUT_DIR",
type=str,
default='',
help="Directory to write all output files,"
" or only advection velocity components if "
"NOWCAST_FILEPATHS is specified.")
parser.add_argument("--nowcast_filepaths",
nargs="+",
type=str,
default=None,
help="Optional list of full paths to "
"output nowcast files. Overrides OUTPUT_DIR. Ignored "
"unless '--extrapolate' is set.")
parser.add_argument("--orographic_enhancement_filepaths",
nargs="+",
type=str,
default=None,
help="List or wildcarded "
"file specification to the input orographic "
"enhancement files. Orographic enhancement files are "
"compulsory for precipitation fields.")
parser.add_argument("--json_file",
metavar="JSON_FILE",
default=None,
help="Filename for the json file containing "
"required changes to the metadata. Information "
"describing the intended contents of the json file "
"is available in "
"improver.utilities.cube_metadata.amend_metadata."
"Every output cube will have the metadata_dict "
"applied. Defaults to None.",
type=str)
# OpticalFlow plugin configurable parameters
parser.add_argument("--ofc_box_size",
type=int,
default=30,
help="Size of "
"square 'box' (in grid squares) within which to solve "
"the optical flow equations.")
parser.add_argument("--smart_smoothing_iterations",
type=int,
default=100,
help="Number of iterations to perform in enforcing "
"smoothness constraint for optical flow velocities.")
# AdvectField options
parser.add_argument("--extrapolate",
action="store_true",
default=False,
help="Optional flag to advect current data forward to "
"specified lead times.")
parser.add_argument("--max_lead_time",
type=int,
default=360,
help="Maximum lead time required (mins). Ignored "
"unless '--extrapolate' is set.")
parser.add_argument("--lead_time_interval",
type=int,
default=15,
help="Interval between required lead times (mins). "
"Ignored unless '--extrapolate' is set.")
args = parser.parse_args(args=argv)
# Load Cubes and JSON.
metadata_dict = load_json_or_none(args.json_file)
original_cube_list = load_cubelist(args.input_filepaths)
oe_cube = load_cube(args.orographic_enhancement_filepaths, allow_none=True)
# Process
forecast_cubes, u_and_v_mean = process(original_cube_list, oe_cube,
metadata_dict, args.ofc_box_size,
args.smart_smoothing_iterations,
args.extrapolate,
args.max_lead_time,
args.lead_time_interval)
# Save Cubes
for wind_cube in u_and_v_mean:
file_name = generate_file_name(wind_cube)
save_netcdf(wind_cube, os.path.join(args.output_dir, file_name))
# advect latest input data to the required lead times
if args.extrapolate:
if args.nowcast_filepaths:
if len(args.nowcast_filepaths) != len(forecast_cubes):
raise ValueError("Require exactly one output file name for "
"each forecast lead time")
for i, cube in enumerate(forecast_cubes):
# save to a suitably-named output file
if args.nowcast_filepaths:
file_name = args.nowcast_filepaths[i]
else:
file_name = os.path.join(args.output_dir,
generate_file_name(cube))
save_netcdf(cube, file_name)
def test_a_cubelist_is_loaded(self):
"""Test that a cubelist is loaded when a valid filepath is provided."""
result = load_cubelist(self.filepath)
self.assertIsInstance(result, iris.cube.CubeList)
def main(argv=None):
"""Load in arguments for estimating coefficients for Ensemble Model Output
Statistics (EMOS), otherwise known as Non-homogeneous Gaussian
Regression (NGR). 2 sources of input data must be provided: historical
forecasts and historical truth data (to use in calibration). The
estimated coefficients are written to a netCDF file.
"""
parser = ArgParser(
description='Estimate coefficients for Ensemble Model Output '
'Statistics (EMOS), otherwise known as Non-homogeneous '
'Gaussian Regression (NGR). There are two methods for '
'inputting data into this CLI, either by providing the '
'historic forecasts and truth separately, or by providing '
'a combined list of historic forecasts and truths along '
'with historic_forecast_identifier and truth_identifier '
'arguments to provide metadata that distinguishes between '
'them.')
parser.add_argument('distribution', metavar='DISTRIBUTION',
choices=['gaussian', 'truncated_gaussian'],
help='The distribution that will be used for '
'calibration. This will be dependent upon the '
'input phenomenon. This has to be supported by '
'the minimisation functions in '
'ContinuousRankedProbabilityScoreMinimisers.')
parser.add_argument('cycletime', metavar='CYCLETIME', type=str,
help='This denotes the cycle at which forecasts '
'will be calibrated using the calculated '
'EMOS coefficients. The validity time in the '
'output coefficients cube will be calculated '
'relative to this cycletime. '
'This cycletime is in the format '
'YYYYMMDDTHHMMZ.')
# Historic forecast and truth filepaths
parser.add_argument(
'--historic_filepath', metavar='HISTORIC_FILEPATH', nargs='+',
help='Paths to the input NetCDF files containing the '
'historic forecast(s) used for calibration. '
'This must be supplied with an associated truth filepath. '
'Specification of either the combined_filepath, '
'historic_forecast_identifier or the truth_identifier is '
'invalid with this argument.')
parser.add_argument(
'--truth_filepath', metavar='TRUTH_FILEPATH', nargs='+',
help='Paths to the input NetCDF files containing the '
'historic truth analyses used for calibration. '
'This must be supplied with an associated historic filepath. '
'Specification of either the combined_filepath, '
'historic_forecast_identifier or the truth_identifier is '
'invalid with this argument.')
# Input filepaths
parser.add_argument(
'--combined_filepath', metavar='COMBINED_FILEPATH', nargs='+',
help='Paths to the input NetCDF files containing '
'both the historic forecast(s) and truth '
'analyses used for calibration. '
'This must be supplied with both the '
'historic_forecast_identifier and the truth_identifier. '
'Specification of either the historic_filepath or the '
'truth_filepath is invalid with this argument.')
parser.add_argument(
"--historic_forecast_identifier",
metavar='HISTORIC_FORECAST_IDENTIFIER',
help='The path to a json file containing metadata '
'information that defines the historic forecast. '
'This must be supplied with both the combined_filepath and the '
'truth_identifier. Specification of either the historic_filepath'
'or the truth_filepath is invalid with this argument. '
'The intended contents is described in improver.'
'ensemble_calibration.ensemble_calibration_utilities.'
'SplitHistoricForecastAndTruth.')
parser.add_argument(
"--truth_identifier", metavar='TRUTH_IDENTIFIER',
help='The path to a json file containing metadata '
'information that defines the truth.'
'This must be supplied with both the combined_filepath and the '
'historic_forecast_identifier. Specification of either the '
'historic_filepath or the truth_filepath is invalid with this '
'argument. The intended contents is described in improver.'
'ensemble_calibration.ensemble_calibration_utilities.'
'SplitHistoricForecastAndTruth.')
# Output filepath
parser.add_argument('output_filepath', metavar='OUTPUT_FILEPATH',
help='The output path for the processed NetCDF')
# Optional arguments.
parser.add_argument('--units', metavar='UNITS',
help='The units that calibration should be undertaken '
'in. The historical forecast and truth will be '
'converted as required.')
parser.add_argument('--predictor_of_mean', metavar='PREDICTOR_OF_MEAN',
choices=['mean', 'realizations'], default='mean',
help='String to specify the predictor used to '
'calibrate the forecast mean. Currently the '
'ensemble mean ("mean") and the ensemble '
'realizations ("realizations") are supported as '
'options. Default: "mean".')
parser.add_argument('--max_iterations', metavar='MAX_ITERATIONS',
type=np.int32, default=1000,
help='The maximum number of iterations allowed '
'until the minimisation has converged to a '
'stable solution. If the maximum number '
'of iterations is reached, but the '
'minimisation has not yet converged to a '
'stable solution, then the available solution '
'is used anyway, and a warning is raised.'
'This may be modified for testing purposes '
'but otherwise kept fixed. If the '
'predictor_of_mean is "realizations", '
'then the number of iterations may require '
'increasing, as there will be more coefficients '
'to solve for.')
args = parser.parse_args(args=argv)
# Load Cubes
historic_forecast = load_cube(args.historic_filepath, allow_none=True)
truth = load_cube(args.truth_filepath, allow_none=True)
combined = (load_cubelist(args.combined_filepath)
if args.combined_filepath else None)
historic_forecast_dict = (
load_json_or_none(args.historic_forecast_identifier))
truth_dict = load_json_or_none(args.truth_identifier)
# Process Cube
coefficients = process(historic_forecast, truth, combined,
historic_forecast_dict, truth_dict,
args.distribution, args.cycletime, args.units,
args.predictor_of_mean, args.max_iterations)
# Save Cube
# Check whether a coefficients cube has been created. If the historic
# forecasts and truths provided did not match in validity time, then
# no coefficients would have been calculated.
if coefficients:
save_netcdf(coefficients, args.output_filepath)
def test_lazy_load(self):
"""Test that the loading works correctly with lazy loading."""
result = load_cubelist([self.filepath, self.filepath])
self.assertIsInstance(result, iris.cube.CubeList)
self.assertArrayEqual([True, True],
[_.has_lazy_data() for _ in result])
def main(argv=None):
"""Load in arguments and ensure they are set correctly.
Then load in the data to blend and calculate default weights
using the method chosen before carrying out the blending."""
parser = ArgParser(
description='Calculate the default weights to apply in weighted '
'blending plugins using the ChooseDefaultWeightsLinear or '
'ChooseDefaultWeightsNonLinear plugins. Then apply these '
'weights to the dataset using the BasicWeightedAverage plugin.'
' Required for ChooseDefaultWeightsLinear: y0val and ynval.'
' Required for ChooseDefaultWeightsNonLinear: cval.'
' Required for ChooseWeightsLinear with dict: wts_dict.')
parser.add_argument('--wts_calc_method',
metavar='WEIGHTS_CALCULATION_METHOD',
choices=['linear', 'nonlinear', 'dict'],
default='linear',
help='Method to use to calculate '
'weights used in blending. "linear" (default): '
'calculate linearly varying blending weights. '
'"nonlinear": calculate blending weights that decrease'
' exponentially with increasing blending coordinate. '
'"dict": calculate weights using a dictionary passed '
'in as a command line argument.')
parser.add_argument('coordinate',
type=str,
metavar='COORDINATE_TO_AVERAGE_OVER',
help='The coordinate over which the blending '
'will be applied.')
parser.add_argument('--coordinate_unit',
metavar='UNIT_STRING',
default='hours since 1970-01-01 00:00:00',
help='Units for blending coordinate. Default= '
'hours since 1970-01-01 00:00:00')
parser.add_argument('--calendar',
metavar='CALENDAR',
help='Calendar for time coordinate. Default=gregorian')
parser.add_argument('--cycletime',
metavar='CYCLETIME',
type=str,
help='The forecast reference time to be used after '
'blending has been applied, in the format '
'YYYYMMDDTHHMMZ. If not provided, the blended file '
'will take the latest available forecast reference '
'time from the input cubes supplied.')
parser.add_argument('--model_id_attr',
metavar='MODEL_ID_ATTR',
type=str,
default="mosg__model_configuration",
help='The name of the netCDF file attribute to be '
'used to identify the source model for '
'multi-model blends. Default assumes Met Office '
'model metadata. Must be present on all input '
'files if blending over models.')
parser.add_argument('--spatial_weights_from_mask',
action='store_true',
default=False,
help='If set this option will result in the generation'
' of spatially varying weights based on the'
' masks of the data we are blending. The'
' one dimensional weights are first calculated '
' using the chosen weights calculation method,'
' but the weights will then be adjusted spatially'
' based on where there is masked data in the data'
' we are blending. The spatial weights are'
' calculated using the'
' SpatiallyVaryingWeightsFromMask plugin.')
parser.add_argument('weighting_mode',
metavar='WEIGHTED_BLEND_MODE',
choices=['weighted_mean', 'weighted_maximum'],
help='The method used in the weighted blend. '
'"weighted_mean": calculate a normal weighted'
' mean across the coordinate. '
'"weighted_maximum": multiplies the values in the'
' coordinate by the weights, and then takes the'
' maximum.')
parser.add_argument('input_filepaths',
metavar='INPUT_FILES',
nargs="+",
help='Paths to input files to be blended.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
spatial = parser.add_argument_group(
'Spatial weights from mask options',
'Options for calculating the spatial weights using the '
'SpatiallyVaryingWeightsFromMask plugin.')
spatial.add_argument('--fuzzy_length',
metavar='FUZZY_LENGTH',
type=float,
default=20000,
help='When calculating spatially varying weights we'
' can smooth the weights so that areas close to'
' areas that are masked have lower weights than'
' those further away. This fuzzy length controls'
' the scale over which the weights are smoothed.'
' The fuzzy length is in terms of m, the'
' default is 20km. This distance is then'
' converted into a number of grid squares,'
' which does not have to be an integer. Assumes'
' the grid spacing is the same in the x and y'
' directions, and raises an error if this is not'
' true. See SpatiallyVaryingWeightsFromMask for'
' more detail.')
linear = parser.add_argument_group(
'linear weights options', 'Options for the linear weights '
'calculation in '
'ChooseDefaultWeightsLinear')
linear.add_argument('--y0val',
metavar='LINEAR_STARTING_POINT',
type=float,
help='The relative value of the weighting start point '
'(lowest value of blend coord) for choosing default '
'linear weights. This must be a positive float or 0.')
linear.add_argument('--ynval',
metavar='LINEAR_END_POINT',
type=float,
help='The relative value of the weighting '
'end point (highest value of blend coord) for choosing'
' default linear weights. This must be a positive '
'float or 0. Note that if blending over forecast '
'reference time, ynval >= y0val would normally be '
'expected (to give greater weight to the more recent '
'forecast).')
nonlinear = parser.add_argument_group(
'nonlinear weights options', 'Options for the non-linear '
'weights calculation in '
'ChooseDefaultWeightsNonLinear')
nonlinear.add_argument('--cval',
metavar='NON_LINEAR_FACTOR',
type=float,
help='Factor used to determine how skewed the '
'non linear weights will be. '
'A value of 1 implies equal weighting. If not '
'set, a default value of cval=0.85 is set.')
wts_dict = parser.add_argument_group(
'dict weights options', 'Options for linear weights to be '
'calculated based on parameters '
'read from a json file dict')
wts_dict.add_argument('--wts_dict',
metavar='WEIGHTS_DICTIONARY',
help='Path to json file containing dictionary from '
'which to calculate blending weights. Dictionary '
'format is as specified in the improver.blending.'
'weights.ChooseWeightsLinear plugin.')
wts_dict.add_argument('--weighting_coord',
metavar='WEIGHTING_COORD',
default='forecast_period',
help='Name of '
'coordinate over which linear weights should be '
'scaled. This coordinate must be avilable in the '
'weights dictionary.')
args = parser.parse_args(args=argv)
# if the linear weights method is called with non-linear args or vice
# versa, exit with error
if (args.wts_calc_method == "linear") and args.cval:
parser.wrong_args_error('cval', 'linear')
if ((args.wts_calc_method == "nonlinear")
and np.any([args.y0val, args.ynval])):
parser.wrong_args_error('y0val, ynval', 'non-linear')
if (args.wts_calc_method == "dict") and not args.wts_dict:
parser.error('Dictionary is required if --wts_calc_method="dict"')
# set blending coordinate units
if "time" in args.coordinate:
coord_unit = Unit(args.coordinate_unit, args.calendar)
elif args.coordinate_unit != 'hours since 1970-01-01 00:00:00.':
coord_unit = args.coordinate_unit
else:
coord_unit = 'no_unit'
# For blending across models, only blending across "model_id" is directly
# supported. This is because the blending coordinate must be sortable, in
# order to ensure that the data cube and the weights cube have coordinates
# in the same order for blending. Whilst the model_configuration is
# sortable itself, as it is associated with model_id, which is the
# dimension coordinate, sorting the model_configuration coordinate can
# result in the model_id coordinate becoming non-monotonic. As dimension
# coordinates must be monotonic, this leads to the model_id coordinate
# being demoted to an auxiliary coordinate. Therefore, for simplicity
# model_id is used as the blending coordinate, instead of
# model_configuration.
# TODO: Support model_configuration as a blending coordinate directly.
if args.coordinate == "model_configuration":
blend_coord = "model_id"
dict_coord = "model_configuration"
else:
blend_coord = args.coordinate
dict_coord = args.coordinate
# load cubes to be blended
cubelist = load_cubelist(args.input_filepaths)
# determine whether or not to equalise forecast periods for model
# blending weights calculation
weighting_coord = (args.weighting_coord
if args.weighting_coord else "forecast_period")
# prepare cubes for weighted blending
merger = MergeCubesForWeightedBlending(blend_coord,
weighting_coord=weighting_coord,
model_id_attr=args.model_id_attr)
cube = merger.process(cubelist, cycletime=args.cycletime)
# if the coord for blending does not exist or has only one value,
# update metadata only
coord_names = [coord.name() for coord in cube.coords()]
if (blend_coord not in coord_names) or (len(
cube.coord(blend_coord).points) == 1):
result = cube.copy()
conform_metadata(result, cube, blend_coord, cycletime=args.cycletime)
# raise a warning if this happened because the blend coordinate
# doesn't exist
if blend_coord not in coord_names:
warnings.warn('Blend coordinate {} is not present on input '
'data'.format(blend_coord))
# otherwise, calculate weights and blend across specified dimension
else:
weights = calculate_blending_weights(
cube,
blend_coord,
args.wts_calc_method,
wts_dict=args.wts_dict,
weighting_coord=args.weighting_coord,
coord_unit=coord_unit,
y0val=args.y0val,
ynval=args.ynval,
cval=args.cval,
dict_coord=dict_coord)
if args.spatial_weights_from_mask:
check_if_grid_is_equal_area(cube)
grid_cells_x, _ = convert_distance_into_number_of_grid_cells(
cube, args.fuzzy_length, int_grid_cells=False)
SpatialWeightsPlugin = SpatiallyVaryingWeightsFromMask(
grid_cells_x)
weights = SpatialWeightsPlugin.process(cube, weights, blend_coord)
# blend across specified dimension
BlendingPlugin = WeightedBlendAcrossWholeDimension(
blend_coord, args.weighting_mode, cycletime=args.cycletime)
result = BlendingPlugin.process(cube, weights=weights)
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Calculate optical flow advection velocities and (optionally)
extrapolate data."""
parser = ArgParser(
description="Calculate optical flow components from input fields "
"and (optionally) extrapolate to required lead times.")
parser.add_argument("input_filepaths", metavar="INPUT_FILEPATHS",
nargs=3, type=str, help="Paths to the input radar "
"files. There should be 3 input files at T, T-1 and "
"T-2 from which to calculate optical flow velocities. "
"The files require a 'time' coordinate on which they "
"are sorted, so the order of inputs does not matter.")
parser.add_argument("--output_dir", metavar="OUTPUT_DIR", type=str,
default='', help="Directory to write all output files,"
" or only advection velocity components if "
"NOWCAST_FILEPATHS is specified.")
parser.add_argument("--nowcast_filepaths", nargs="+", type=str,
default=None, help="Optional list of full paths to "
"output nowcast files. Overrides OUTPUT_DIR. Ignored "
"unless '--extrapolate' is set.")
parser.add_argument("--orographic_enhancement_filepaths", nargs="+",
type=str, default=None, help="List or wildcarded "
"file specification to the input orographic "
"enhancement files. Orographic enhancement files are "
"compulsory for precipitation fields.")
parser.add_argument("--json_file", metavar="JSON_FILE", default=None,
help="Filename for the json file containing "
"required changes to the metadata. Information "
"describing the intended contents of the json file "
"is available in "
"improver.utilities.cube_metadata.amend_metadata."
"Every output cube will have the metadata_dict "
"applied. Defaults to None.", type=str)
# OpticalFlow plugin configurable parameters
parser.add_argument("--ofc_box_size", type=int, default=30, help="Size of "
"square 'box' (in grid squares) within which to solve "
"the optical flow equations.")
parser.add_argument("--smart_smoothing_iterations", type=int, default=100,
help="Number of iterations to perform in enforcing "
"smoothness constraint for optical flow velocities.")
# AdvectField options
parser.add_argument("--extrapolate", action="store_true", default=False,
help="Optional flag to advect current data forward to "
"specified lead times.")
parser.add_argument("--max_lead_time", type=int, default=360,
help="Maximum lead time required (mins). Ignored "
"unless '--extrapolate' is set.")
parser.add_argument("--lead_time_interval", type=int, default=15,
help="Interval between required lead times (mins). "
"Ignored unless '--extrapolate' is set.")
args = parser.parse_args(args=argv)
# read input data
original_cube_list = load_cubelist(args.input_filepaths)
if args.orographic_enhancement_filepaths:
# Subtract orographic enhancement
oe_cube = load_cube(args.orographic_enhancement_filepaths)
cube_list = ApplyOrographicEnhancement("subtract").process(
original_cube_list, oe_cube)
else:
cube_list = original_cube_list
if any("precipitation_rate" in cube.name() for cube in cube_list):
cube_names = [cube.name() for cube in cube_list]
msg = ("For precipitation fields, orographic enhancement "
"filepaths must be supplied. The names of the cubes "
"supplied were: {}".format(cube_names))
raise ValueError(msg)
# order input files by validity time
cube_list.sort(key=lambda x: x.coord("time").points[0])
time_coord = cube_list[-1].coord("time")
metadata_dict = None
if args.json_file:
# Load JSON file for metadata amendments.
with open(args.json_file, 'r') as input_file:
metadata_dict = json.load(input_file)
# calculate optical flow velocities from T-1 to T and T-2 to T-1
ofc_plugin = OpticalFlow(iterations=args.smart_smoothing_iterations,
metadata_dict=metadata_dict)
ucubes = iris.cube.CubeList([])
vcubes = iris.cube.CubeList([])
for older_cube, newer_cube in zip(cube_list[:-1], cube_list[1:]):
ucube, vcube = ofc_plugin.process(older_cube, newer_cube,
boxsize=args.ofc_box_size)
ucubes.append(ucube)
vcubes.append(vcube)
# average optical flow velocity components
ucube = ucubes.merge_cube()
umean = ucube.collapsed("time", iris.analysis.MEAN)
umean.coord("time").points = time_coord.points
umean.coord("time").units = time_coord.units
vcube = vcubes.merge_cube()
vmean = vcube.collapsed("time", iris.analysis.MEAN)
vmean.coord("time").points = time_coord.points
vmean.coord("time").units = time_coord.units
# save mean optical flow components as netcdf files
for wind_cube in [umean, vmean]:
file_name = generate_file_name(wind_cube)
save_netcdf(wind_cube, os.path.join(args.output_dir, file_name))
# advect latest input data to the required lead times
if args.extrapolate:
# generate list of lead times in minutes
lead_times = np.arange(0, args.max_lead_time+1,
args.lead_time_interval)
if args.nowcast_filepaths:
if len(args.nowcast_filepaths) != len(lead_times):
raise ValueError("Require exactly one output file name for "
"each forecast lead time")
forecast_plugin = CreateExtrapolationForecast(
original_cube_list[-1], umean, vmean,
orographic_enhancement_cube=oe_cube, metadata_dict=metadata_dict)
# extrapolate input data to required lead times
for i, lead_time in enumerate(lead_times):
forecast_cube = forecast_plugin.extrapolate(
leadtime_minutes=lead_time)
# save to a suitably-named output file
if args.nowcast_filepaths:
file_name = args.nowcast_filepaths[i]
else:
file_name = os.path.join(
args.output_dir, generate_file_name(forecast_cube))
save_netcdf(forecast_cube, file_name)
