def main(argv=None):
"""Load in arguments and get going."""
parser = ArgParser(
description="Calculate percentiled data over a given coordinate by "
"collapsing that coordinate. Typically used to convert realization "
"data into percentiled data, but may calculate over any "
"dimension coordinate. Alternatively, calling this CLI with a dataset"
" containing probabilities will convert those to percentiles using "
"the ensemble copula coupling plugin. If no particular percentiles "
"are given at which to calculate values and no 'number of percentiles'"
" to calculate are specified, the following defaults will be used: "
"[0, 5, 10, 20, 25, 30, 40, 50, 60, 70, 75, 80, 90, 95, 100]")
parser.add_argument("input_filepath",
metavar="INPUT_FILE",
help="A path to an input NetCDF file to be processed")
parser.add_argument("output_filepath",
metavar="OUTPUT_FILE",
help="The output path for the processed NetCDF")
parser.add_argument("--coordinates",
metavar="COORDINATES_TO_COLLAPSE",
nargs="+",
help="Coordinate or coordinates over which to collapse"
" data and calculate percentiles; e.g. "
"'realization' or 'latitude longitude'. This argument "
"must be provided when collapsing a coordinate or "
"coordinates to create percentiles, but is redundant "
"when converting probabilities to percentiles and may "
"be omitted. This coordinate(s) will be removed "
"and replaced by a percentile coordinate.")
parser.add_argument('--ecc_bounds_warning',
default=False,
action='store_true',
help='If True, where calculated percentiles are '
'outside the ECC bounds range, raise a warning '
'rather than an exception.')
group = parser.add_mutually_exclusive_group(required=False)
group.add_argument("--percentiles",
metavar="PERCENTILES",
nargs="+",
default=None,
type=float,
help="Optional definition of percentiles at which to "
"calculate data, e.g. --percentiles 0 33.3 66.6 100")
group.add_argument('--no-of-percentiles',
default=None,
type=int,
metavar='NUMBER_OF_PERCENTILES',
help="Optional definition of the number of percentiles "
"to be generated, these distributed regularly with the "
"aim of dividing into blocks of equal probability.")
args = parser.parse_args(args=argv)
cube = load_cube(args.input_filepath)
percentiles = args.percentiles
if args.no_of_percentiles is not None:
percentiles = choose_set_of_percentiles(args.no_of_percentiles,
sampling="quantile")
# TODO: Correct when formal cf-standards exists
if 'probability_of_' in cube.name():
if args.coordinates:
warnings.warn("Converting probabilities to percentiles. The "
"provided COORDINATES_TO_COLLAPSE variable will "
"not be used.")
result = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=args.ecc_bounds_warning).process(
cube, percentiles=percentiles)
else:
if not args.coordinates:
raise ValueError("To collapse a coordinate to calculate "
"percentiles, a coordinate or list of "
"coordinates must be provided.")
# Switch back to use the slow scipy method if the cube contains masked
# data which the numpy method cannot handle.
fast_percentile_method = True
if np.ma.is_masked(cube.data):
# Check for masked points:
fast_percentile_method = False
elif np.ma.isMaskedArray(cube.data):
# Check if we have a masked array with an empty mask. If so,
# replace it with a non-masked array:
cube.data = cube.data.data
result = PercentileConverter(
args.coordinates,
percentiles=percentiles,
fast_percentile_method=fast_percentile_method).process(cube)
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Load in arguments and get going."""
parser = ArgParser(
description="Calculate percentiled data over a given coordinate by "
"collapsing that coordinate. Typically used to convert realization "
"data into percentiled data, but may calculate over any "
"dimension coordinate. Alternatively, calling this CLI with a dataset"
" containing probabilities will convert those to percentiles using "
"the ensemble copula coupling plugin. If no particular percentiles "
"are given at which to calculate values and no 'number of percentiles'"
" to calculate are specified, the following defaults will be used: "
"[0, 5, 10, 20, 25, 30, 40, 50, 60, 70, 75, 80, 90, 95, 100]")
parser.add_argument("input_filepath",
metavar="INPUT_FILE",
help="A path to an input NetCDF file to be processed")
parser.add_argument("output_filepath",
metavar="OUTPUT_FILE",
help="The output path for the processed NetCDF")
parser.add_argument("--coordinates",
metavar="COORDINATES_TO_COLLAPSE",
nargs="+",
help="Coordinate or coordinates over which to collapse"
" data and calculate percentiles; e.g. "
"'realization' or 'latitude longitude'. This argument "
"must be provided when collapsing a coordinate or "
"coordinates to create percentiles, but is redundant "
"when converting probabilities to percentiles and may "
"be omitted. This coordinate(s) will be removed "
"and replaced by a percentile coordinate.")
parser.add_argument('--ecc_bounds_warning',
default=False,
action='store_true',
help='If True, where calculated percentiles are '
'outside the ECC bounds range, raise a warning '
'rather than an exception.')
group = parser.add_mutually_exclusive_group(required=False)
group.add_argument("--percentiles",
metavar="PERCENTILES",
nargs="+",
default=None,
type=float,
help="Optional definition of percentiles at which to "
"calculate data, e.g. --percentiles 0 33.3 66.6 100")
group.add_argument('--no-of-percentiles',
default=None,
type=int,
metavar='NUMBER_OF_PERCENTILES',
help="Optional definition of the number of percentiles "
"to be generated, these distributed regularly with the "
"aim of dividing into blocks of equal probability.")
args = parser.parse_args(args=argv)
# Load Cube
cube = load_cube(args.input_filepath)
# Process Cube
result = process(cube, args.coordinates, args.ecc_bounds_warning,
args.percentiles, args.no_of_percentiles)
# Save Cube
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Load in arguments for applying neighbourhood processing when using a
mask."""
from improver.argparser import ArgParser
parser = ArgParser(
description='Apply the requested neighbourhood method via the '
'ApplyNeighbourhoodProcessingWithAMask plugin to a file '
'with one diagnostic dataset in combination with a file '
'containing one or more masks. The mask dataset may have '
'an extra dimension compared to the input diagnostic. '
'In this case, the user specifies the name of '
'the extra coordinate and this coordinate is iterated '
'over so each mask is applied to separate slices over the'
' input data. These intermediate masked datasets are then'
' concatenated, resulting in a dataset that has been '
' processed using multiple masks and has gained an extra '
'dimension from the masking. There is also an option to '
're-mask the output dataset, so that after '
'neighbourhood processing, non-zero values are only '
'present for unmasked grid points. '
'There is an alternative option of collapsing the '
'dimension that we gain using this processing using a '
'weighted average.')
parser.add_argument('coord_for_masking', metavar='COORD_FOR_MASKING',
help='Coordinate to iterate over when applying a mask '
'to the neighbourhood processing. ')
parser.add_argument('input_filepath', metavar='INPUT_FILE',
help='A path to an input NetCDF file to be processed.')
parser.add_argument('input_mask_filepath', metavar='INPUT_MASK_FILE',
help='A path to an input mask NetCDF file to be '
'used to mask the input file.')
parser.add_argument('output_filepath', metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
group = parser.add_mutually_exclusive_group()
group.add_argument('--radius', metavar='RADIUS', type=float,
help='The radius (in m) for neighbourhood processing.')
group.add_argument('--radii-by-lead-time',
metavar=('RADII_BY_LEAD_TIME', 'LEAD_TIME_IN_HOURS'),
nargs=2,
help='The radii for neighbourhood processing '
'and the associated lead times at which the radii are '
'valid. The radii are in metres whilst the lead time '
'has units of hours. The radii and lead times are '
'expected as individual comma-separated lists with '
'the list of radii given first followed by a list of '
'lead times to indicate at what lead time each radii '
'should be used. For example: 10000,12000,14000 1,2,3 '
'where a lead time of 1 hour uses a radius of 10000m, '
'a lead time of 2 hours uses a radius of 12000m, etc.')
parser.add_argument('--sum_or_fraction', default="fraction",
choices=["sum", "fraction"],
help='The neighbourhood output can either be in the '
'form of a sum of the neighbourhood, or a '
'fraction calculated by dividing the sum of the '
'neighbourhood by the neighbourhood area. '
'"fraction" is the default option.')
group2 = parser.add_mutually_exclusive_group()
group2.add_argument('--re_mask', action='store_true',
help='If re_mask is set (i.e. True), the output data '
'following neighbourhood processing is '
're-masked. This re-masking removes any values '
'that have been generated by neighbourhood '
'processing at grid points that were '
'originally masked. '
'If not set, re_mask defaults to False and no '
're-masking is applied to the neighbourhood '
'processed output. Therefore, the neighbourhood '
'processing may result in values being present '
'in areas that were originally masked. This '
'allows the the values in adjacent bands to be'
'weighted together if the additional dimension'
'from the masking process is collapsed.')
group2.add_argument('--collapse_dimension', action='store_true',
help='Collapse the dimension from the mask, by doing '
'a weighted mean using the weights provided. '
'This is only suitable when the result is is '
'left unmasked, so there is data to weight '
'between the points in coordinate we are '
'collapsing.')
parser.add_argument('--weights_for_collapsing_dim', metavar='WEIGHTS',
default=None,
help='A path to an weights NetCDF file containing the '
'weights which are used for collapsing the '
'dimension gained through masking.')
parser.add_argument('--intermediate_filepath', default=None,
help='If provided the result after neighbourhooding, '
'before collapsing the extra dimension is saved '
'in the given filepath.')
args = parser.parse_args(args=argv)
# Load Cubes
cube = cli.inputcube(args.input_filepath)
mask_cube = cli.inputcube(args.input_mask_filepath)
weights = cli.inputcube(args.weights_for_collapsing_dim) if \
args.collapse_dimension else None
# Process Cube
# pylint: disable=E1123
process(cube, mask_cube, weights, coord_for_masking=args.coord_for_masking,
radius=args.radius, radii_by_lead_time=args.radii_by_lead_time,
sum_or_fraction=args.sum_or_fraction, remask=args.re_mask,
collapse_dimension=args.collapse_dimension,
output=args.output_filepath,
intermediate_output=args.intermediate_filepath)
def main(argv=None):
"""Load in arguments and get going."""
parser = ArgParser(
description='Apply the requested neighbourhood method via '
'the NeighbourhoodProcessing plugin to a file '
'whose data can be loaded as a single iris.cube.Cube.')
parser.add_argument(
'neighbourhood_output',
metavar='NEIGHBOURHOOD_OUTPUT',
help='The form of the results generated using neighbourhood '
'processing. If "probabilities" is selected, the mean '
'probability within a neighbourhood is calculated. If '
'"percentiles" is selected, then the percentiles are calculated '
'within a neighbourhood. Calculating percentiles from a '
'neighbourhood is only supported for a circular neighbourhood. '
'Options: "probabilities", "percentiles".')
parser.add_argument('neighbourhood_shape',
metavar='NEIGHBOURHOOD_SHAPE',
choices=["circular", "square"],
help='The shape of the neighbourhood to apply in '
'neighbourhood processing. Only a "circular" '
'neighbourhood shape is applicable for '
'calculating "percentiles" output. '
'Options: "circular", "square".')
group = parser.add_mutually_exclusive_group()
group.add_argument('--radius',
metavar='RADIUS',
type=float,
help='The radius (in m) for neighbourhood processing.')
group.add_argument('--radii-by-lead-time',
metavar=('RADII_BY_LEAD_TIME', 'LEAD_TIME_IN_HOURS'),
nargs=2,
help='The radii for neighbourhood processing '
'and the associated lead times at which the radii are '
'valid. The radii are in metres whilst the lead time '
'has units of hours. The radii and lead times are '
'expected as individual comma-separated lists with '
'the list of radii given first followed by a list of '
'lead times to indicate at what lead time each radii '
'should be used. For example: 10000,12000,14000 1,2,3 '
'where a lead time of 1 hour uses a radius of 10000m, '
'a lead time of 2 hours uses a radius of 12000m, etc.')
parser.add_argument('--degrees_as_complex',
action='store_true',
default=False,
help='Set this flag to process angles,'
' eg wind directions, as complex numbers. Not '
'compatible with circular kernel, percentiles or '
'recursive filter.')
parser.add_argument('--weighted_mode',
action='store_true',
default=False,
help='For neighbourhood processing using a circular '
'kernel, setting the weighted_mode indicates the '
'weighting decreases with radius. '
'If weighted_mode is not set, a constant '
'weighting is assumed. weighted_mode is only '
'applicable for calculating "probability" '
'neighbourhood output.')
parser.add_argument('--sum_or_fraction',
default="fraction",
choices=["sum", "fraction"],
help='The neighbourhood output can either be in the '
'form of a sum of the neighbourhood, or a '
'fraction calculated by dividing the sum of the '
'neighbourhood by the neighbourhood area. '
'"fraction" is the default option.')
parser.add_argument('--re_mask',
action='store_true',
help='If re_mask is set (i.e. True), the original '
'un-neighbourhood processed mask is applied to '
'mask out the neighbourhood processed dataset. '
'If not set, re_mask defaults to False and the '
'original un-neighbourhood processed mask is '
'not applied. Therefore, the neighbourhood '
'processing may result in values being present '
'in areas that were originally masked. ')
parser.add_argument('--percentiles',
metavar='PERCENTILES',
default=DEFAULT_PERCENTILES,
nargs='+',
type=float,
help='Calculate values at the specified percentiles '
'from the neighbourhood surrounding each grid '
'point.')
parser.add_argument('input_filepath',
metavar='INPUT_FILE',
help='A path to an input NetCDF file to be processed.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
parser.add_argument('--input_mask_filepath',
metavar='INPUT_MASK_FILE',
help='A path to an input mask NetCDF file to be '
'used to mask the input file. '
'This is currently only supported for square '
'neighbourhoods. The data should contain 1 for '
'usable points and 0 for discarded points, e.g. '
'a land-mask.')
parser.add_argument('--halo_radius',
metavar='HALO_RADIUS',
default=None,
type=float,
help='radius in metres of excess halo to clip.'
' Used where a larger'
' grid was defined than the standard grid'
' and we want to clip the grid back to the'
' standard grid e.g. for global data'
' regridded to UK area. Default=None')
parser.add_argument('--apply-recursive-filter',
action='store_true',
default=False,
help='Option to apply the recursive filter to a '
'square neighbourhooded output dataset, '
'converting it into a Gaussian-like kernel or '
'smoothing over short distances. '
'The filter uses an alpha '
'parameter (0 < alpha < 1) to control what '
'proportion of the probability is passed onto '
'the next grid-square in the x and y directions. '
'The alpha parameter can be set on a grid-square '
'by grid-square basis for the x and y directions '
'separately (using two arrays of alpha '
'parameters of the same dimensionality as the '
'domain). Alternatively a single alpha value can '
'be set for each of the x and y directions. These'
' methods can be mixed, e.g. an array for the x '
'direction and a float for the y direction and '
'vice versa. The recursive filter cannot be '
'applied to a circular kernel')
parser.add_argument('--input_filepath_alphas_x_cube',
metavar='ALPHAS_X_FILE',
help='A path to a NetCDF file describing the alpha '
'factors to be used for smoothing in the x '
'direction when applying the recursive filter')
parser.add_argument('--input_filepath_alphas_y_cube',
metavar='ALPHAS_Y_FILE',
help='A path to a NetCDF file describing the alpha '
'factors to be used for smoothing in the y '
'direction when applying the recursive filter')
parser.add_argument('--alpha_x',
metavar='ALPHA_X',
default=None,
type=float,
help='A single alpha factor (0 < alpha_x < 1) to be '
'applied to every grid square in the x '
'direction when applying the recursive filter')
parser.add_argument('--alpha_y',
metavar='ALPHA_Y',
default=None,
type=float,
help='A single alpha factor (0 < alpha_y < 1) to be '
'applied to every grid square in the y '
'direction when applying the recursive filter.')
parser.add_argument('--iterations',
metavar='ITERATIONS',
default=1,
type=int,
help='Number of times to apply the filter, default=1 '
'(typically < 5)')
args = parser.parse_args(args=argv)
if (args.neighbourhood_output == "percentiles"
and args.neighbourhood_shape == "square"):
parser.wrong_args_error('square', 'neighbourhood_shape')
if (args.neighbourhood_output == "percentiles" and args.weighted_mode):
parser.wrong_args_error('weighted_mode',
'neighbourhood_shape=percentiles')
if (args.neighbourhood_output == "probabilities"
and args.percentiles != DEFAULT_PERCENTILES):
parser.wrong_args_error('percentiles',
'neighbourhood_shape=probabilities')
if (args.input_mask_filepath and args.neighbourhood_shape == "circular"):
parser.wrong_args_error('neighbourhood_shape=circular',
'input_mask_filepath')
if args.degrees_as_complex:
if args.neighbourhood_output == "percentiles":
parser.error('Cannot generate percentiles from complex numbers')
if args.neighbourhood_shape == "circular":
parser.error('Cannot process complex numbers with circular '
'neighbourhoods')
if args.apply_recursive_filter:
parser.error('Cannot process complex numbers with recursive '
'filter')
cube = load_cube(args.input_filepath)
if args.degrees_as_complex:
# convert cube data into complex numbers
cube.data = WindDirection.deg_to_complex(cube.data)
if args.radius:
radius_or_radii = args.radius
lead_times = None
elif args.radii_by_lead_time:
radius_or_radii = args.radii_by_lead_time[0].split(",")
lead_times = args.radii_by_lead_time[1].split(",")
if args.input_mask_filepath:
mask_cube = load_cube(args.input_mask_filepath)
else:
mask_cube = None
if args.neighbourhood_output == "probabilities":
result = (NeighbourhoodProcessing(args.neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
weighted_mode=args.weighted_mode,
sum_or_fraction=args.sum_or_fraction,
re_mask=args.re_mask).process(
cube, mask_cube=mask_cube))
elif args.neighbourhood_output == "percentiles":
result = (GeneratePercentilesFromANeighbourhood(
args.neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
percentiles=args.percentiles).process(cube))
# If the '--apply-recursive-filter' option has been specified in the
# input command, pass the neighbourhooded 'result' cube obtained above
# through the recursive-filter plugin before saving the output.
# The recursive filter is only applicable to square neighbourhoods.
if args.neighbourhood_shape == 'square' and args.apply_recursive_filter:
alphas_x_cube = None
alphas_y_cube = None
if args.input_filepath_alphas_x_cube is not None:
alphas_x_cube = load_cube(args.input_filepath_alphas_x_cube)
if args.input_filepath_alphas_y_cube is not None:
alphas_y_cube = load_cube(args.input_filepath_alphas_y_cube)
result = RecursiveFilter(alpha_x=args.alpha_x,
alpha_y=args.alpha_y,
iterations=args.iterations,
re_mask=args.re_mask).process(
result,
alphas_x=alphas_x_cube,
alphas_y=alphas_y_cube,
mask_cube=mask_cube)
elif args.neighbourhood_shape == 'circular' and \
args.apply_recursive_filter:
raise ValueError('Recursive filter option is not applicable to '
'circular neighbourhoods. ')
if args.degrees_as_complex:
# convert neighbourhooded cube back to degrees
result.data = WindDirection.complex_to_deg(result.data)
if args.halo_radius is not None:
result = remove_cube_halo(result, args.halo_radius)
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Extrapolate data forward in time."""
parser = ArgParser(
description="Extrapolate input data to required lead times.")
parser.add_argument("input_filepath", metavar="INPUT_FILEPATH",
type=str, help="Path to input NetCDF file.")
group = parser.add_mutually_exclusive_group()
group.add_argument("--output_dir", metavar="OUTPUT_DIR", type=str,
default="", help="Directory to write output files.")
group.add_argument("--output_filepaths", nargs="+", type=str,
help="List of full paths to output nowcast files, in "
"order of increasing lead time.")
optflw = parser.add_argument_group('Advect using files containing the x '
' and y components of the velocity')
optflw.add_argument("--eastward_advection_filepath", type=str, help="Path"
" to input file containing Eastward advection "
"velocities.")
optflw.add_argument("--northward_advection_filepath", type=str, help="Path"
" to input file containing Northward advection "
"velocities.")
speed = parser.add_argument_group('Advect using files containing speed and'
' direction')
speed.add_argument("--advection_speed_filepath", type=str, help="Path"
" to input file containing advection speeds,"
" usually wind speeds, on multiple pressure levels.")
speed.add_argument("--advection_direction_filepath", type=str,
help="Path to input file containing the directions from"
" which advection speeds are coming (180 degrees from"
" the direction in which the speed is directed). The"
" directions should be on the same grid as the input"
" speeds, including the same vertical levels.")
speed.add_argument("--pressure_level", type=int, default=75000, help="The"
" pressure level in Pa to extract from the multi-level"
" advection_speed and advection_direction files. The"
" velocities at this level are used for advection.")
parser.add_argument("--orographic_enhancement_filepaths", nargs="+",
type=str, default=None, help="List or wildcarded "
"file specification to the input orographic "
"enhancement files. Orographic enhancement files are "
"compulsory for precipitation fields.")
parser.add_argument("--json_file", metavar="JSON_FILE", default=None,
help="Filename for the json file containing "
"required changes to the metadata. Information "
"describing the intended contents of the json file "
"is available in "
"improver.utilities.cube_metadata.amend_metadata."
"Every output cube will have the metadata_dict "
"applied. Defaults to None.", type=str)
parser.add_argument("--max_lead_time", type=int, default=360,
help="Maximum lead time required (mins).")
parser.add_argument("--lead_time_interval", type=int, default=15,
help="Interval between required lead times (mins).")
accumulation_args = parser.add_argument_group(
'Calculate accumulations from advected fields')
accumulation_args.add_argument(
"--accumulation_fidelity", type=int, default=0,
help="If set, this CLI will additionally return accumulations"
" calculated from the advected fields. This fidelity specifies the"
" time interval in minutes between advected fields that is used to"
" calculate these accumulations. This interval must be a factor of"
" the lead_time_interval.")
accumulation_args.add_argument(
"--accumulation_units", type=str, default='m',
help="Desired units in which the accumulations should be expressed,"
"e.g. mm")
args = parser.parse_args(args=argv)
upath, vpath = (args.eastward_advection_filepath,
args.northward_advection_filepath)
spath, dpath = (args.advection_speed_filepath,
args.advection_direction_filepath)
# load files and initialise advection plugin
input_cube = load_cube(args.input_filepath)
if (upath and vpath) and not (spath or dpath):
ucube = load_cube(upath)
vcube = load_cube(vpath)
elif (spath and dpath) and not (upath or vpath):
level_constraint = Constraint(pressure=args.pressure_level)
try:
scube = load_cube(spath, constraints=level_constraint)
dcube = load_cube(dpath, constraints=level_constraint)
except ValueError as err:
raise ValueError(
'{} Unable to extract specified pressure level from given '
'speed and direction files.'.format(err))
ucube, vcube = ResolveWindComponents().process(scube, dcube)
else:
raise ValueError('Cannot mix advection component velocities with speed'
' and direction')
oe_cube = None
if args.orographic_enhancement_filepaths:
oe_cube = load_cube(args.orographic_enhancement_filepaths)
metadata_dict = None
if args.json_file:
# Load JSON file for metadata amendments.
with open(args.json_file, 'r') as input_file:
metadata_dict = json.load(input_file)
# generate list of lead times in minutes
lead_times = np.arange(0, args.max_lead_time+1,
args.lead_time_interval)
if args.output_filepaths:
if len(args.output_filepaths) != len(lead_times):
raise ValueError("Require exactly one output file name for each "
"forecast lead time")
# determine whether accumulations are also to be returned.
time_interval = args.lead_time_interval
if args.accumulation_fidelity > 0:
fraction, _ = np.modf(args.lead_time_interval /
args.accumulation_fidelity)
if fraction != 0:
msg = ("The specified lead_time_interval ({}) is not cleanly "
"divisible by the specified accumulation_fidelity ({}). As "
"a result the lead_time_interval cannot be constructed from"
" accumulation cubes at this fidelity.".format(
args.lead_time_interval, args.accumulation_fidelity))
raise ValueError(msg)
time_interval = args.accumulation_fidelity
lead_times = np.arange(0, args.max_lead_time+1, time_interval)
lead_time_filter = args.lead_time_interval // time_interval
forecast_plugin = CreateExtrapolationForecast(
input_cube, ucube, vcube, orographic_enhancement_cube=oe_cube,
metadata_dict=metadata_dict)
# extrapolate input data to required lead times
forecast_cubes = iris.cube.CubeList()
for i, lead_time in enumerate(lead_times):
forecast_cubes.append(
forecast_plugin.extrapolate(leadtime_minutes=lead_time))
# return rate cubes
for i, cube in enumerate(forecast_cubes[::lead_time_filter]):
# save to a suitably-named output file
if args.output_filepaths:
file_name = args.output_filepaths[i]
else:
file_name = os.path.join(
args.output_dir, generate_file_name(cube))
save_netcdf(cube, file_name)
# calculate accumulations if required
if args.accumulation_fidelity > 0:
plugin = Accumulation(accumulation_units=args.accumulation_units,
accumulation_period=args.lead_time_interval * 60)
accumulation_cubes = plugin.process(forecast_cubes)
# return accumulation cubes
for i, cube in enumerate(accumulation_cubes):
file_name = os.path.join(args.output_dir, generate_file_name(cube))
save_netcdf(cube, file_name)
def main(argv=None):
"""
Load in the arguments and apply the requested variant of Ensemble
Copula Coupling for converting percentile data to realizations.
"""
parser = ArgParser(
description='Convert a dataset containing '
'probabilities into one containing '
'ensemble realizations using Ensemble Copula Coupling.')
# General options:
parser.add_argument('input_filepath', metavar='INPUT_FILE',
help='A path to an input NetCDF file to be processed.'
' Must contain a percentile dimension.')
parser.add_argument('output_filepath', metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
parser.add_argument('--no_of_percentiles', default=None, type=int,
metavar='NUMBER_OF_PERCENTILES',
help='The number of percentiles to be generated. '
'This is also equal to the number of ensemble '
'realizations that will be generated.')
parser.add_argument('--sampling_method', default='quantile',
const='quantile', nargs='?',
choices=['quantile', 'random'],
metavar='PERCENTILE_SAMPLING_METHOD',
help='Method to be used for generating the list of '
'percentiles with forecasts generated at each '
'percentile. The options are "quantile" and '
'"random". "quantile" is the default option. '
'The "quantile" option produces equally spaced '
'percentiles which is the preferred '
'option for full Ensemble Copula Coupling with '
'reordering enabled.')
parser.add_argument(
'--ecc_bounds_warning', default=False, action='store_true',
help='If True, where percentiles (calculated as an intermediate '
'output before realizations) exceed the ECC bounds range, raise '
'a warning rather than an exception.')
# Different use cases:
# (We can either reorder OR rebadge)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--reordering', default=False, action='store_true',
help='The option used to create ensemble realizations '
'from percentiles by reordering the input '
'percentiles based on the order of the '
'raw ensemble forecast.')
group.add_argument('--rebadging', default=False, action='store_true',
help='The option used to create ensemble realizations '
'from percentiles by rebadging the input '
'percentiles.')
# If reordering, can do so either based on original realizations,
# or randomly.
reordering = parser.add_argument_group(
'Reordering options', 'Options for reordering the input percentiles '
'using the raw ensemble forecast as required to create ensemble '
'realizations.')
reordering.add_argument('--raw_forecast_filepath',
metavar='RAW_FORECAST_FILE',
help='A path to an raw forecast NetCDF file to be '
'processed. This option is compulsory, if the '
'reordering option is selected.')
reordering.add_argument('--random_ordering', default=False,
action='store_true',
help='Decide whether or not to use random '
'ordering within the ensemble reordering step.')
reordering.add_argument(
'--random_seed', default=None,
help='Option to specify a value for the random seed for testing '
'purposes, otherwise, the default random seed behaviour is '
'utilised. The random seed is used in the generation of the '
'random numbers used for either the random_ordering option to '
'order the input percentiles randomly, rather than use the '
'ordering from the raw ensemble, or for splitting tied values '
'within the raw ensemble, so that the values from the input '
'percentiles can be ordered to match the raw ensemble.')
rebadging = parser.add_argument_group(
'Rebadging options', 'Options for rebadging the input percentiles '
'as ensemble realizations.')
rebadging.add_argument('--realization_numbers', default=None,
metavar='REALIZATION_NUMBERS', nargs="+",
help='A list of ensemble realization numbers to '
'use when rebadging the percentiles '
'into realizations.')
args = parser.parse_args(args=argv)
# CLI argument checking:
# Can only do one of reordering or rebadging: if options are passed which
# correspond to the opposite method, raise an exception.
# Note: Shouldn't need to check that both/none are set, since they are
# defined as mandatory, but mutually exclusive, options.
if args.reordering:
if args.realization_numbers is not None:
parser.wrong_args_error('realization_numbers', 'reordering')
if args.rebadging:
if ((args.raw_forecast_filepath is not None) or
(args.random_ordering is not False)):
parser.wrong_args_error(
'raw_forecast_filepath, random_ordering', 'rebadging')
# Convert the string of realization_numbers to a list of ints.
realization_numbers = None
if args.rebadging:
if args.realization_numbers is not None:
realization_numbers = (
[int(num) for num in args.realization_numbers])
cube = load_cube(args.input_filepath)
raw_forecast = load_cube(args.raw_forecast_filepath, allow_none=True)
# Process Cube
result_cube = process(cube, raw_forecast, args.no_of_percentiles,
args.sampling_method, args.ecc_bounds_warning,
args.reordering, args.rebadging,
args.random_ordering, args.random_seed,
realization_numbers)
# Save Cube
save_netcdf(result_cube, args.output_filepath)
def main(argv=None):
"""Convert from probabilities to ensemble realizations via a CLI."""
cli_specific_arguments = [(['--no_of_realizations'], {
'metavar':
'NUMBER_OF_REALIZATIONS',
'default':
None,
'type':
int,
'help':
("Optional definition of the number of ensemble realizations to "
"be generated. These are generated through an intermediate "
"percentile representation. These percentiles will be "
"distributed regularly with the aim of dividing into blocks of "
"equal probability. If the reordering option is specified and "
"the number of realizations is not given then the number of "
"realizations is taken from the number of realizations in the "
"raw forecast NetCDF file.")
})]
cli_definition = {
'central_arguments': ('input_file', 'output_file'),
'specific_arguments':
cli_specific_arguments,
'description': ('Convert a dataset containing '
'probabilities into one containing '
'ensemble realizations.')
}
parser = ArgParser(**cli_definition)
# add mutually exculsive options rebadge and reorder.
# If reordering add option for raw ensemble - raise error if
# raw ens missing.
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--reordering',
default=False,
action='store_true',
help='The option used to create ensemble realizations '
'from percentiles by reordering the input '
'percentiles based on the order of the '
'raw ensemble forecast.')
group.add_argument('--rebadging',
default=False,
action='store_true',
help='The option used to create ensemble realizations '
'from percentiles by rebadging the input '
'percentiles.')
# If reordering, we need a raw ensemble forecast.
reordering = parser.add_argument_group(
'Reordering options', 'Options for reordering the input percentiles '
'using the raw ensemble forecast as required to create ensemble '
'realizations.')
reordering.add_argument('--raw_forecast_filepath',
metavar='RAW_FORECAST_FILE',
help='A path to an raw forecast NetCDF file to be '
'processed. This option is compulsory, if the '
'reordering option is selected.')
reordering.add_argument(
'--random_seed',
default=None,
help='Option to specify a value for the random seed for testing '
'purposes, otherwise, the default random seed behaviour is '
'utilised. The random seed is used in the generation of the '
'random numbers used for splitting tied values '
'within the raw ensemble, so that the values from the input '
'percentiles can be ordered to match the raw ensemble.')
reordering.add_argument(
'--ecc_bounds_warning',
default=False,
action='store_true',
help='If True, where percentiles (calculated as an intermediate '
'output before realizations) exceed the ECC bounds range, raise '
'a warning rather than an exception.')
args = parser.parse_args(args=argv)
# CLI argument checking:
# Can only do one of reordering or rebadging: if options are passed which
# correspond to the opposite method, raise an exception.
# Note: Shouldn't need to check that both/none are set, since they are
# defined as mandatory, but mutually exclusive, options.
if args.rebadging:
if ((args.raw_forecast_filepath is not None)
or (args.random_seed is not None)):
parser.wrong_args_error('raw_forecast_filepath, random_seed',
'rebadging')
# Process the data
cube = load_cube(args.input_filepath)
if args.reordering:
if args.raw_forecast_filepath is None:
message = ("You must supply a raw forecast filepath if using the "
"reordering option.")
raise ValueError(message)
else:
raw_forecast = load_cube(args.raw_forecast_filepath)
try:
raw_forecast.coord("realization")
except CoordinateNotFoundError:
message = ("The netCDF file from the raw_forecast_filepath "
"must have a realization coordinate.")
raise ValueError(message)
no_of_realizations = args.no_of_realizations
# If no_of_realizations is not given, take the number from the raw
# ensemble cube.
if args.no_of_realizations is None:
no_of_realizations = len(raw_forecast.coord("realization").points)
cube = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=args.ecc_bounds_warning).process(
cube, no_of_percentiles=no_of_realizations)
cube = EnsembleReordering().process(cube,
raw_forecast,
random_ordering=False,
random_seed=args.random_seed)
elif args.rebadging:
cube = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=args.ecc_bounds_warning).process(
cube, no_of_percentiles=args.no_of_realizations)
cube = RebadgePercentilesAsRealizations().process(cube)
save_netcdf(cube, args.output_filepath)
def main(argv=None):
"""
Interpolate data to intermediate times between the validity times of two
cubes. This can be used to fill in missing data (e.g. for radar fields) or
to ensure data is available at the required intervals when model data is
not available at these times.
"""
parser = ArgParser(description='Interpolate data between validity times ')
parser.add_argument('infiles',
metavar='INFILES',
nargs=2,
help='Files contain the data at the beginning'
' and end of the period (2 files required).')
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("--interval_in_mins",
metavar="INTERVAL_IN_MINS",
default=None,
type=int,
help="Specifies the interval in minutes"
" at which to interpolate "
"between the two input cubes."
" A number of minutes which does not "
"divide up the interval equally will "
"raise an exception. If intervals_in_mins "
"is set then times can not be set.")
group.add_argument("--times",
metavar="TIMES",
default=None,
nargs="+",
type=str,
help="Specifies the times in the format "
"{YYYYMMDD}T{HHMM}Z "
" at which to interpolate "
"between the two input cubes."
"Where {YYYYMMDD} is year, month day "
"and {HHMM} is hour and minutes e.g "
"20180116T0100Z. More than one time"
"can be provided separated by a space "
"but if times are set interval_in_mins "
"can not be set")
parser.add_argument("--interpolation_method",
metavar="INTERPOLATION_METHOD",
default="linear",
choices=["linear", "solar", "daynight"],
help="Specifies the interpolation method; "
"solar interpolates using the solar elevation, "
"daynight uses linear interpolation but sets"
" night time points to 0.0, "
"linear is linear interpolation. "
"Default is linear.")
parser.add_argument("--output_files",
metavar="OUTPUT_FILES",
required=True,
nargs="+",
help="List of output files."
" The interpolated files will always be"
" in the chronological order of"
" earliest to latest "
" regardless of the order of the infiles.")
args = parser.parse_args(args=argv)
# Load Cubes
cube_0 = load_cube(args.infiles[0])
cube_1 = load_cube(args.infiles[1])
# Process Cubes
interpolated_cubes = process(cube_0, cube_1, args.interval_in_mins,
args.times, args.interpolation_method)
# Save Cubes
len_files = len(args.output_files)
len_cubes = len(interpolated_cubes)
if len_files == len_cubes:
for i, cube_out in enumerate(interpolated_cubes):
save_netcdf(cube_out, args.output_files[i])
else:
msg = ("Output_files do not match cubes created. "
"{} files given but {} required.".format(len_files, len_cubes))
raise ValueError(msg)
def main(argv=None):
"""Extrapolate data forward in time."""
parser = ArgParser(
description="Extrapolate input data to required lead times.")
parser.add_argument("input_filepath",
metavar="INPUT_FILEPATH",
type=str,
help="Path to input NetCDF file.")
group = parser.add_mutually_exclusive_group()
group.add_argument("--output_dir",
metavar="OUTPUT_DIR",
type=str,
default="",
help="Directory to write output files.")
group.add_argument("--output_filepaths",
nargs="+",
type=str,
help="List of full paths to output nowcast files, in "
"order of increasing lead time.")
optflw = parser.add_argument_group('Advect using files containing the x '
' and y components of the velocity')
optflw.add_argument("--eastward_advection_filepath",
type=str,
help="Path"
" to input file containing Eastward advection "
"velocities.")
optflw.add_argument("--northward_advection_filepath",
type=str,
help="Path"
" to input file containing Northward advection "
"velocities.")
speed = parser.add_argument_group('Advect using files containing speed and'
' direction')
speed.add_argument("--advection_speed_filepath",
type=str,
help="Path"
" to input file containing advection speeds,"
" usually wind speeds, on multiple pressure levels.")
speed.add_argument("--advection_direction_filepath",
type=str,
help="Path to input file containing the directions from"
" which advection speeds are coming (180 degrees from"
" the direction in which the speed is directed). The"
" directions should be on the same grid as the input"
" speeds, including the same vertical levels.")
speed.add_argument("--pressure_level",
type=int,
default=75000,
help="The"
" pressure level in Pa to extract from the multi-level"
" advection_speed and advection_direction files. The"
" velocities at this level are used for advection.")
parser.add_argument("--orographic_enhancement_filepaths",
nargs="+",
type=str,
default=None,
help="List or wildcarded "
"file specification to the input orographic "
"enhancement files. Orographic enhancement files are "
"compulsory for precipitation fields.")
parser.add_argument("--json_file",
metavar="JSON_FILE",
default=None,
help="Filename for the json file containing "
"required changes to the metadata. Information "
"describing the intended contents of the json file "
"is available in "
"improver.utilities.cube_metadata.amend_metadata."
"Every output cube will have the metadata_dict "
"applied. Defaults to None.",
type=str)
parser.add_argument("--max_lead_time",
type=int,
default=360,
help="Maximum lead time required (mins).")
parser.add_argument("--lead_time_interval",
type=int,
default=15,
help="Interval between required lead times (mins).")
accumulation_args = parser.add_argument_group(
'Calculate accumulations from advected fields')
accumulation_args.add_argument(
"--accumulation_fidelity",
type=int,
default=0,
help="If set, this CLI will additionally return accumulations"
" calculated from the advected fields. This fidelity specifies the"
" time interval in minutes between advected fields that is used to"
" calculate these accumulations. This interval must be a factor of"
" the lead_time_interval.")
accumulation_args.add_argument(
"--accumulation_period",
type=int,
default=15,
help="The period over which the accumulation is calculated (mins). "
"Only full accumulation periods will be computed. At lead times "
"that are shorter than the accumulation period, no accumulation "
"output will be produced.")
accumulation_args.add_argument(
"--accumulation_units",
type=str,
default='m',
help="Desired units in which the accumulations should be expressed,"
"e.g. mm")
# Load Cubes
args = parser.parse_args(args=argv)
metadata_dict = load_json_or_none(args.json_file)
upath, vpath = (args.eastward_advection_filepath,
args.northward_advection_filepath)
spath, dpath = (args.advection_speed_filepath,
args.advection_direction_filepath)
# load files and initialise advection plugin
input_cube = load_cube(args.input_filepath)
orographic_enhancement_cube = load_cube(
args.orographic_enhancement_filepaths, allow_none=True)
speed_cube = direction_cube = ucube = vcube = None
if (upath and vpath) and not (spath or dpath):
ucube = load_cube(upath)
vcube = load_cube(vpath)
elif (spath and dpath) and not (upath or vpath):
level_constraint = Constraint(pressure=args.pressure_level)
try:
speed_cube = load_cube(spath, constraints=level_constraint)
direction_cube = load_cube(dpath, constraints=level_constraint)
except ValueError as err:
raise ValueError(
'{} Unable to extract specified pressure level from given '
'speed and direction files.'.format(err))
else:
raise ValueError('Cannot mix advection component velocities with speed'
' and direction')
# Process Cubes
accumulation_cubes, forecast_to_return = process(
input_cube, ucube, vcube, speed_cube, direction_cube,
orographic_enhancement_cube, metadata_dict, args.max_lead_time,
args.lead_time_interval, args.accumulation_fidelity,
args.accumulation_period, args.accumulation_units)
# Save Cube
if args.output_filepaths and \
len(args.output_filepaths) != len(forecast_to_return):
raise ValueError("Require exactly one output file name for each "
"forecast lead time")
for i, cube in enumerate(forecast_to_return):
# save to a suitably-named output file
if args.output_filepaths:
file_name = args.output_filepaths[i]
else:
file_name = os.path.join(args.output_dir, generate_file_name(cube))
save_netcdf(cube, file_name)
if args.accumulation_fidelity > 0:
# return accumulation cubes
for i, cube in enumerate(accumulation_cubes):
file_name = os.path.join(args.output_dir, generate_file_name(cube))
save_netcdf(cube, file_name)
