def main(argv=None):
"""Load in arguments for estimating coefficients for Ensemble Model Output
Statistics (EMOS), otherwise known as Non-homogeneous Gaussian
Regression (NGR). 2 sources of input data must be provided: historical
forecasts and historical truth data (to use in calibration). The
estimated coefficients are written to a netCDF file.
"""
parser = ArgParser(
description='Estimate coefficients for Ensemble Model Output '
'Statistics (EMOS), otherwise known as Non-homogeneous '
'Gaussian Regression (NGR). There are two methods for '
'inputting data into this CLI, either by providing the '
'historic forecasts and truth separately, or by providing '
'a combined list of historic forecasts and truths along '
'with historic_forecast_identifier and truth_identifier '
'arguments to provide metadata that distinguishes between '
'them.')
parser.add_argument('distribution', metavar='DISTRIBUTION',
choices=['gaussian', 'truncated_gaussian'],
help='The distribution that will be used for '
'calibration. This will be dependent upon the '
'input phenomenon. This has to be supported by '
'the minimisation functions in '
'ContinuousRankedProbabilityScoreMinimisers.')
parser.add_argument('cycletime', metavar='CYCLETIME', type=str,
help='This denotes the cycle at which forecasts '
'will be calibrated using the calculated '
'EMOS coefficients. The validity time in the '
'output coefficients cube will be calculated '
'relative to this cycletime. '
'This cycletime is in the format '
'YYYYMMDDTHHMMZ.')
# Historic forecast and truth filepaths
parser.add_argument(
'--historic_filepath', metavar='HISTORIC_FILEPATH', nargs='+',
help='Paths to the input NetCDF files containing the '
'historic forecast(s) used for calibration. '
'This must be supplied with an associated truth filepath. '
'Specification of either the combined_filepath, '
'historic_forecast_identifier or the truth_identifier is '
'invalid with this argument.')
parser.add_argument(
'--truth_filepath', metavar='TRUTH_FILEPATH', nargs='+',
help='Paths to the input NetCDF files containing the '
'historic truth analyses used for calibration. '
'This must be supplied with an associated historic filepath. '
'Specification of either the combined_filepath, '
'historic_forecast_identifier or the truth_identifier is '
'invalid with this argument.')
# Input filepaths
parser.add_argument(
'--combined_filepath', metavar='COMBINED_FILEPATH', nargs='+',
help='Paths to the input NetCDF files containing '
'both the historic forecast(s) and truth '
'analyses used for calibration. '
'This must be supplied with both the '
'historic_forecast_identifier and the truth_identifier. '
'Specification of either the historic_filepath or the '
'truth_filepath is invalid with this argument.')
parser.add_argument(
"--historic_forecast_identifier",
metavar='HISTORIC_FORECAST_IDENTIFIER',
help='The path to a json file containing metadata '
'information that defines the historic forecast. '
'This must be supplied with both the combined_filepath and the '
'truth_identifier. Specification of either the historic_filepath'
'or the truth_filepath is invalid with this argument. '
'The intended contents is described in improver.'
'ensemble_calibration.ensemble_calibration_utilities.'
'SplitHistoricForecastAndTruth.')
parser.add_argument(
"--truth_identifier", metavar='TRUTH_IDENTIFIER',
help='The path to a json file containing metadata '
'information that defines the truth.'
'This must be supplied with both the combined_filepath and the '
'historic_forecast_identifier. Specification of either the '
'historic_filepath or the truth_filepath is invalid with this '
'argument. The intended contents is described in improver.'
'ensemble_calibration.ensemble_calibration_utilities.'
'SplitHistoricForecastAndTruth.')
# Output filepath
parser.add_argument('output_filepath', metavar='OUTPUT_FILEPATH',
help='The output path for the processed NetCDF')
# Optional arguments.
parser.add_argument('--units', metavar='UNITS',
help='The units that calibration should be undertaken '
'in. The historical forecast and truth will be '
'converted as required.')
parser.add_argument('--predictor_of_mean', metavar='PREDICTOR_OF_MEAN',
choices=['mean', 'realizations'], default='mean',
help='String to specify the predictor used to '
'calibrate the forecast mean. Currently the '
'ensemble mean ("mean") and the ensemble '
'realizations ("realizations") are supported as '
'options. Default: "mean".')
parser.add_argument('--max_iterations', metavar='MAX_ITERATIONS',
type=np.int32, default=1000,
help='The maximum number of iterations allowed '
'until the minimisation has converged to a '
'stable solution. If the maximum number '
'of iterations is reached, but the '
'minimisation has not yet converged to a '
'stable solution, then the available solution '
'is used anyway, and a warning is raised.'
'This may be modified for testing purposes '
'but otherwise kept fixed. If the '
'predictor_of_mean is "realizations", '
'then the number of iterations may require '
'increasing, as there will be more coefficients '
'to solve for.')
args = parser.parse_args(args=argv)
# Load Cubes
historic_forecast = load_cube(args.historic_filepath, allow_none=True)
truth = load_cube(args.truth_filepath, allow_none=True)
combined = (load_cubelist(args.combined_filepath)
if args.combined_filepath else None)
historic_forecast_dict = (
load_json_or_none(args.historic_forecast_identifier))
truth_dict = load_json_or_none(args.truth_identifier)
# Process Cube
coefficients = process(historic_forecast, truth, combined,
historic_forecast_dict, truth_dict,
args.distribution, args.cycletime, args.units,
args.predictor_of_mean, args.max_iterations)
# Save Cube
# Check whether a coefficients cube has been created. If the historic
# forecasts and truths provided did not match in validity time, then
# no coefficients would have been calculated.
if coefficients:
save_netcdf(coefficients, args.output_filepath)
def main(argv=None):
"""Load in arguments and get going."""
parser = ArgParser(
description=('Reads input orography and landmask fields. Creates '
'a series of topographic zone weights to indicate '
'where an orography point sits within the defined '
'topographic bands. If the orography point is in the '
'centre of a topographic band, then a single band will '
'have a weight of 1.0. If the orography point is at the '
'edge of a topographic band, then the upper band will '
'have a 0.5 weight whilst the lower band will also have '
'a 0.5 weight. Otherwise, the weight will vary linearly '
'between the centre of a topographic band and the edge.'))
parser.add_argument('input_filepath_standard_orography',
metavar='INPUT_FILE_STANDARD_OROGRAPHY',
help=('A path to an input NetCDF orography file to '
'be processed'))
parser.add_argument('output_filepath', metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
parser.add_argument('--input_filepath_landmask', metavar='INPUT_FILE_LAND',
help=('A path to an input NetCDF land mask file to be '
'processed. If provided, sea points will be '
'masked and set to the default fill value. If '
'no land mask is provided, weights will be '
'generated for sea points as well as land, '
'included in the appropriate topographic band.'))
parser.add_argument('--force', dest='force', default=False,
action='store_true',
help=('If keyword is set (i.e. True), ancillaries '
'will be generated even if doing so will '
'overwrite existing files'))
parser.add_argument('--thresholds_filepath',
metavar='THRESHOLDS_FILEPATH',
default=None,
help=("The path to a json file which can be used "
"to set the number and size of topographic "
"bounds. If unset a default bounds dictionary"
" will be used:"
"{'bounds': [[-500., 50.], [50., 100.], "
"[100., 150.],[150., 200.], [200., 250.], "
"[250., 300.], [300., 400.], [400., 500.], "
"[500., 650.],[650., 800.], [800., 950.], "
"[950., 6000.]], 'units': 'm'}"))
args = parser.parse_args(args=argv)
thresholds_dict = load_json_or_none(args.thresholds_filepath)
if not os.path.exists(args.output_filepath) or args.force:
orography = load_cube(args.input_filepath_standard_orography)
landmask = None
if args.input_filepath_landmask:
try:
landmask = load_cube(args.input_filepath_landmask)
except IOError as err:
msg = ("Loading land mask has been unsuccessful: {}. "
"This may be because the land mask could not be "
"located in {}; run "
'improver-generate-landmask-ancillary first.').format(
err, args.input_filepath_landmask)
raise IOError(msg)
result = process(landmask, orography, thresholds_dict)
# Save Cube
save_netcdf(result, args.output_filepath)
else:
print('File already exists here: ', args.output_filepath)
def test_none(self, m):
"""Tests if called with None returns None."""
file_path = None
dict_read = load_json_or_none(file_path)
self.assertIsNone(dict_read)
m.assert_not_called()
def main(argv=None):
"""Load in arguments for the cube combiner plugin.
"""
parser = ArgParser(
description="Combine the input files into a single file using "
"the requested operation e.g. + - min max etc.")
parser.add_argument("input_filenames", metavar="INPUT_FILENAMES",
nargs="+", type=str,
help="Paths to the input NetCDF files. Each input"
" file should be able to be loaded as a single "
" iris.cube.Cube instance. The resulting file"
" metadata will be based on the first file but"
" its metadata can be overwritten via"
" the metadata_jsonfile option.")
parser.add_argument("output_filepath", metavar="OUTPUT_FILE",
help="The output path for the processed NetCDF.")
parser.add_argument("--operation", metavar="OPERATION",
default="+",
choices=["+", "-", "*",
"add", "subtract", "multiply",
"min", "max", "mean"],
help="Operation to use in combining NetCDF datasets"
" Default=+ i.e. add ", type=str)
parser.add_argument("--new-name", metavar="NEW_NAME",
default=None,
help="New name for the resulting dataset. Will"
" default to the name of the first dataset if "
"not set.", type=str)
parser.add_argument("--metadata_jsonfile", metavar="METADATA_JSONFILE",
default=None,
help="Filename for the json file containing "
"required changes to the metadata. "
" default=None", type=str)
parser.add_argument('--warnings_on', action='store_true',
help="If warnings_on is set (i.e. True), "
"Warning messages where metadata do not match "
"will be given. Default=False", default=False)
args = parser.parse_args(args=argv)
new_metadata = load_json_or_none(args.metadata_jsonfile)
# Load cubes
cubelist = iris.cube.CubeList([])
new_cube_name = args.new_name
for filename in args.input_filenames:
new_cube = load_cube(filename)
cubelist.append(new_cube)
if new_cube_name is None:
new_cube_name = new_cube.name()
if args.warnings_on:
if (args.new_name is None and
new_cube_name != new_cube.name()):
msg = ("Defaulting to first cube name, {} but combining with"
"a cube with name, {}.".format(
new_cube_name, new_cube.name()))
warnings.warn(msg)
# Process Cube
result = process(cubelist, args.operation, new_cube_name,
new_metadata, args.warnings_on)
# Save Cube
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Load in arguments and ensure they are set correctly.
Then load in the data to blend and calculate default weights
using the method chosen before carrying out the blending."""
parser = ArgParser(
description='Calculate the default weights to apply in weighted '
'blending plugins using the ChooseDefaultWeightsLinear or '
'ChooseDefaultWeightsNonLinear plugins. Then apply these '
'weights to the dataset using the BasicWeightedAverage plugin.'
' Required for ChooseDefaultWeightsLinear: y0val and ynval.'
' Required for ChooseDefaultWeightsNonLinear: cval.'
' Required for ChooseWeightsLinear with dict: wts_dict.')
parser.add_argument('--wts_calc_method',
metavar='WEIGHTS_CALCULATION_METHOD',
choices=['linear', 'nonlinear', 'dict'],
default='linear',
help='Method to use to calculate '
'weights used in blending. "linear" (default): '
'calculate linearly varying blending weights. '
'"nonlinear": calculate blending weights that decrease'
' exponentially with increasing blending coordinate. '
'"dict": calculate weights using a dictionary passed '
'in as a command line argument.')
parser.add_argument('coordinate',
type=str,
metavar='COORDINATE_TO_AVERAGE_OVER',
help='The coordinate over which the blending '
'will be applied.')
parser.add_argument('--cycletime',
metavar='CYCLETIME',
type=str,
help='The forecast reference time to be used after '
'blending has been applied, in the format '
'YYYYMMDDTHHMMZ. If not provided, the blended file '
'will take the latest available forecast reference '
'time from the input cubes supplied.')
parser.add_argument('--model_id_attr',
metavar='MODEL_ID_ATTR',
type=str,
default=None,
help='The name of the netCDF file attribute to be '
'used to identify the source model for '
'multi-model blends. Default is None. '
'Must be present on all input '
'files if blending over models.')
parser.add_argument('--spatial_weights_from_mask',
action='store_true',
default=False,
help='If set this option will result in the generation'
' of spatially varying weights based on the'
' masks of the data we are blending. The'
' one dimensional weights are first calculated '
' using the chosen weights calculation method,'
' but the weights will then be adjusted spatially'
' based on where there is masked data in the data'
' we are blending. The spatial weights are'
' calculated using the'
' SpatiallyVaryingWeightsFromMask plugin.')
parser.add_argument('input_filepaths',
metavar='INPUT_FILES',
nargs="+",
help='Paths to input files to be blended.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
spatial = parser.add_argument_group(
'Spatial weights from mask options',
'Options for calculating the spatial weights using the '
'SpatiallyVaryingWeightsFromMask plugin.')
spatial.add_argument('--fuzzy_length',
metavar='FUZZY_LENGTH',
type=float,
default=20000,
help='When calculating spatially varying weights we'
' can smooth the weights so that areas close to'
' areas that are masked have lower weights than'
' those further away. This fuzzy length controls'
' the scale over which the weights are smoothed.'
' The fuzzy length is in terms of m, the'
' default is 20km. This distance is then'
' converted into a number of grid squares,'
' which does not have to be an integer. Assumes'
' the grid spacing is the same in the x and y'
' directions, and raises an error if this is not'
' true. See SpatiallyVaryingWeightsFromMask for'
' more detail.')
linear = parser.add_argument_group(
'linear weights options', 'Options for the linear weights '
'calculation in '
'ChooseDefaultWeightsLinear')
linear.add_argument('--y0val',
metavar='LINEAR_STARTING_POINT',
type=float,
help='The relative value of the weighting start point '
'(lowest value of blend coord) for choosing default '
'linear weights. This must be a positive float or 0.')
linear.add_argument('--ynval',
metavar='LINEAR_END_POINT',
type=float,
help='The relative value of the weighting '
'end point (highest value of blend coord) for choosing'
' default linear weights. This must be a positive '
'float or 0. Note that if blending over forecast '
'reference time, ynval >= y0val would normally be '
'expected (to give greater weight to the more recent '
'forecast).')
nonlinear = parser.add_argument_group(
'nonlinear weights options', 'Options for the non-linear '
'weights calculation in '
'ChooseDefaultWeightsNonLinear')
nonlinear.add_argument('--cval',
metavar='NON_LINEAR_FACTOR',
type=float,
help='Factor used to determine how skewed the '
'non linear weights will be. A value of 1 '
'implies equal weighting.')
wts_dict = parser.add_argument_group(
'dict weights options', 'Options for linear weights to be '
'calculated based on parameters '
'read from a json file dict')
wts_dict.add_argument('--wts_dict',
metavar='WEIGHTS_DICTIONARY',
help='Path to json file containing dictionary from '
'which to calculate blending weights. Dictionary '
'format is as specified in the improver.blending.'
'weights.ChooseWeightsLinear plugin.')
wts_dict.add_argument('--weighting_coord',
metavar='WEIGHTING_COORD',
default='forecast_period',
help='Name of '
'coordinate over which linear weights should be '
'scaled. This coordinate must be available in the '
'weights dictionary.')
args = parser.parse_args(args=argv)
# reject incorrect argument combinations
if (args.wts_calc_method == "linear") and args.cval:
parser.wrong_args_error('cval', 'linear')
if ((args.wts_calc_method == "nonlinear")
and np.any([args.y0val, args.ynval])):
parser.wrong_args_error('y0val, ynval', 'non-linear')
if (args.wts_calc_method == "dict") and not args.wts_dict:
parser.error('Dictionary is required if --wts_calc_method="dict"')
weights_dict = load_json_or_none(args.wts_dict)
# Load cubes to be blended.
cubelist = load_cubelist(args.input_filepaths)
result = process(cubelist, args.wts_calc_method, args.coordinate,
args.cycletime, args.weighting_coord, weights_dict,
args.y0val, args.ynval, args.cval, args.model_id_attr,
args.spatial_weights_from_mask, args.fuzzy_length)
save_netcdf(result, args.output_filepath)
def test_loading_file(self, m):
"""Tests if called with a filepath, loads a dict."""
file_path = "filename"
dict_read = load_json_or_none(file_path)
self.assertEqual(dict_read, {"k": "v"})
m.assert_called_with("filename", "r")
def main(argv=None):
"""Calculate optical flow advection velocities"""
parser = ArgParser(
description="Calculate optical flow components from input fields.")
parser.add_argument("input_filepaths",
metavar="INPUT_FILEPATHS",
nargs=3,
type=str,
help="Paths to the input radar "
"files. There should be 3 input files at T, T-1 and "
"T-2 from which to calculate optical flow velocities. "
"The files require a 'time' coordinate on which they "
"are sorted, so the order of inputs does not matter.")
parser.add_argument("output_filepath",
metavar="OUTPUT_FILEPATH",
help="The output path for the resulting NetCDF")
parser.add_argument("--nowcast_filepaths",
nargs="+",
type=str,
default=None,
help="Optional list of full paths to "
"output nowcast files. Overrides OUTPUT_DIR. Ignored "
"unless '--extrapolate' is set.")
parser.add_argument("--orographic_enhancement_filepaths",
nargs="+",
type=str,
default=None,
help="List or wildcarded "
"file specification to the input orographic "
"enhancement files. Orographic enhancement files are "
"compulsory for precipitation fields.")
parser.add_argument("--json_file",
metavar="JSON_FILE",
default=None,
help="Filename for the json file containing "
"required changes to attributes. "
"Every output file will have the attributes_dict "
"applied. Defaults to None.",
type=str)
# OpticalFlow plugin configurable parameters
parser.add_argument("--ofc_box_size",
type=int,
default=30,
help="Size of "
"square 'box' (in grid squares) within which to solve "
"the optical flow equations.")
parser.add_argument("--smart_smoothing_iterations",
type=int,
default=100,
help="Number of iterations to perform in enforcing "
"smoothness constraint for optical flow velocities.")
args = parser.parse_args(args=argv)
# Load Cubes and JSON
attributes_dict = load_json_or_none(args.json_file)
original_cube_list = load_cubelist(args.input_filepaths)
oe_cube = load_cube(args.orographic_enhancement_filepaths, allow_none=True)
# Process
result = process(original_cube_list, oe_cube, attributes_dict,
args.ofc_box_size, args.smart_smoothing_iterations)
# Save Cubes
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Calculate optical flow advection velocities and (optionally)
extrapolate data."""
parser = ArgParser(
description="Calculate optical flow components from input fields "
"and (optionally) extrapolate to required lead times.")
parser.add_argument("input_filepaths",
metavar="INPUT_FILEPATHS",
nargs=3,
type=str,
help="Paths to the input radar "
"files. There should be 3 input files at T, T-1 and "
"T-2 from which to calculate optical flow velocities. "
"The files require a 'time' coordinate on which they "
"are sorted, so the order of inputs does not matter.")
parser.add_argument("--output_dir",
metavar="OUTPUT_DIR",
type=str,
default='',
help="Directory to write all output files,"
" or only advection velocity components if "
"NOWCAST_FILEPATHS is specified.")
parser.add_argument("--nowcast_filepaths",
nargs="+",
type=str,
default=None,
help="Optional list of full paths to "
"output nowcast files. Overrides OUTPUT_DIR. Ignored "
"unless '--extrapolate' is set.")
parser.add_argument("--orographic_enhancement_filepaths",
nargs="+",
type=str,
default=None,
help="List or wildcarded "
"file specification to the input orographic "
"enhancement files. Orographic enhancement files are "
"compulsory for precipitation fields.")
parser.add_argument("--json_file",
metavar="JSON_FILE",
default=None,
help="Filename for the json file containing "
"required changes to the metadata. Information "
"describing the intended contents of the json file "
"is available in "
"improver.utilities.cube_metadata.amend_metadata."
"Every output cube will have the metadata_dict "
"applied. Defaults to None.",
type=str)
# OpticalFlow plugin configurable parameters
parser.add_argument("--ofc_box_size",
type=int,
default=30,
help="Size of "
"square 'box' (in grid squares) within which to solve "
"the optical flow equations.")
parser.add_argument("--smart_smoothing_iterations",
type=int,
default=100,
help="Number of iterations to perform in enforcing "
"smoothness constraint for optical flow velocities.")
# AdvectField options
parser.add_argument("--extrapolate",
action="store_true",
default=False,
help="Optional flag to advect current data forward to "
"specified lead times.")
parser.add_argument("--max_lead_time",
type=int,
default=360,
help="Maximum lead time required (mins). Ignored "
"unless '--extrapolate' is set.")
parser.add_argument("--lead_time_interval",
type=int,
default=15,
help="Interval between required lead times (mins). "
"Ignored unless '--extrapolate' is set.")
args = parser.parse_args(args=argv)
# Load Cubes and JSON.
metadata_dict = load_json_or_none(args.json_file)
original_cube_list = load_cubelist(args.input_filepaths)
oe_cube = load_cube(args.orographic_enhancement_filepaths, allow_none=True)
# Process
forecast_cubes, u_and_v_mean = process(original_cube_list, oe_cube,
metadata_dict, args.ofc_box_size,
args.smart_smoothing_iterations,
args.extrapolate,
args.max_lead_time,
args.lead_time_interval)
# Save Cubes
for wind_cube in u_and_v_mean:
file_name = generate_file_name(wind_cube)
save_netcdf(wind_cube, os.path.join(args.output_dir, file_name))
# advect latest input data to the required lead times
if args.extrapolate:
if args.nowcast_filepaths:
if len(args.nowcast_filepaths) != len(forecast_cubes):
raise ValueError("Require exactly one output file name for "
"each forecast lead time")
for i, cube in enumerate(forecast_cubes):
# save to a suitably-named output file
if args.nowcast_filepaths:
file_name = args.nowcast_filepaths[i]
else:
file_name = os.path.join(args.output_dir,
generate_file_name(cube))
save_netcdf(cube, file_name)
def main(argv=None):
"""
Standardise a source cube. Available options are regridding (bilinear or
nearest-neighbour, optionally with land-mask awareness), updating meta-data
and converting float64 data to float32. A check for float64 data compliance
can be made by only specify a source NetCDF file with no other arguments.
"""
parser = ArgParser(
description='Standardise a source data cube. Three main options are '
'available; fixing float64 data, regridding and updating '
'metadata. If regridding then additional options are '
'available to use bilinear or nearest-neighbour '
'(optionally with land-mask awareness) modes. If only a '
'source file is specified with no other arguments, then '
'an exception will be raised if float64 data are found on '
'the source.')
parser.add_argument('source_data_filepath', metavar='SOURCE_DATA',
help='A cube of data that is to be standardised and '
'optionally fixed for float64 data, regridded '
'and meta data changed')
parser.add_argument("--output_filepath", metavar="OUTPUT_FILE",
default=None,
help="The output path for the processed NetCDF. "
"If only a source file is specified and no "
"output file, then the source will be checked"
"for float64 data.")
regrid_group = parser.add_argument_group("Regridding options")
regrid_group.add_argument(
"--target_grid_filepath", metavar="TARGET_GRID",
help=('If specified then regridding of the source '
'against the target grid is enabled. If also using '
'landmask-aware regridding, then this must be land_binary_mask '
'data.'))
regrid_group.add_argument(
"--regrid_mode", default='bilinear',
choices=['bilinear', 'nearest', 'nearest-with-mask'],
help=('Selects which regridding technique to use. Default uses '
'iris.analysis.Linear(); "nearest" uses Nearest() (Use for less '
'continuous fields, e.g. precipitation.); "nearest-with-mask" '
'ensures that target data are sourced from points with the same '
'mask value (Use for coast-line-dependent variables like '
'temperature).'))
regrid_group.add_argument(
"--extrapolation_mode", default='nanmask',
help='Mode to use for extrapolating data into regions '
'beyond the limits of the source_data domain. '
'Refer to online documentation for iris.analysis. '
'Modes are: '
'extrapolate - The extrapolation points will '
'take their value from the nearest source point. '
'nan - The extrapolation points will be be '
'set to NaN. '
'error - A ValueError exception will be raised, '
'notifying an attempt to extrapolate. '
'mask - The extrapolation points will always be '
'masked, even if the source data is not a '
'MaskedArray. '
'nanmask - If the source data is a MaskedArray '
'the extrapolation points will be masked. '
'Otherwise they will be set to NaN. '
'Defaults to nanmask.')
regrid_group.add_argument(
"--input_landmask_filepath", metavar="INPUT_LANDMASK_FILE",
help=("A path to a NetCDF file describing the land_binary_mask on "
"the source-grid if coastline-aware regridding is required."))
regrid_group.add_argument(
"--landmask_vicinity", metavar="LANDMASK_VICINITY",
default=25000., type=float,
help=("Radius of vicinity to search for a coastline, in metres. "
"Default value; 25000 m"))
parser.add_argument("--fix_float64", action='store_true', default=False,
help="Check and fix cube for float64 data. Without "
"this option an exception will be raised if "
"float64 data is found but no fix applied.")
parser.add_argument("--json_file", metavar="JSON_FILE", default=None,
help='Filename for the json file containing required '
'changes that will be applied '
'to the metadata. Defaults to None.')
args = parser.parse_args(args=argv)
if args.target_grid_filepath or args.json_file or args.fix_float64:
if not args.output_filepath:
msg = ("An argument has been specified that requires an output "
"filepath but none has been provided")
raise ValueError(msg)
if (args.input_landmask_filepath and
"nearest-with-mask" not in args.regrid_mode):
msg = ("Land-mask file supplied without appropriate regrid_mode. "
"Use --regrid_mode=nearest-with-mask.")
raise ValueError(msg)
if args.input_landmask_filepath and not args.target_grid_filepath:
msg = ("Cannot specify input_landmask_filepath without "
"target_grid_filepath")
raise ValueError(msg)
# Load Cube and json
metadata_dict = load_json_or_none(args.json_file)
# source file data path is a mandatory argument
output_data = load_cube(args.source_data_filepath)
target_grid = None
source_landsea = None
if args.target_grid_filepath:
target_grid = load_cube(args.target_grid_filepath)
if args.regrid_mode in ["nearest-with-mask"]:
if not args.input_landmask_filepath:
msg = ("An argument has been specified that requires an input "
"landmask filepath but none has been provided")
raise ValueError(msg)
source_landsea = load_cube(args.input_landmask_filepath)
# Process Cube
output_data = process(output_data, target_grid, source_landsea,
metadata_dict, args.regrid_mode,
args.extrapolation_mode, args.landmask_vicinity,
args.fix_float64)
# Save Cube
if args.output_filepath:
save_netcdf(output_data, args.output_filepath)
def main(argv=None):
"""Load in arguments and start spotdata extraction process."""
parser = ArgParser(
description="Extract diagnostic data from gridded fields for spot data"
" sites. It is possible to apply a temperature lapse rate adjustment"
" to temperature data that helps to account for differences between"
" the spot sites real altitude and that of the grid point from which"
" the temperature data is extracted.")
# Input and output files required.
parser.add_argument("neighbour_filepath",
metavar="NEIGHBOUR_FILEPATH",
help="Path to a NetCDF file of spot-data neighbours. "
"This file also contains the spot site information.")
parser.add_argument("diagnostic_filepath",
metavar="DIAGNOSTIC_FILEPATH",
help="Path to a NetCDF file containing the diagnostic "
"data to be extracted.")
parser.add_argument("temperature_lapse_rate_filepath",
metavar="LAPSE_RATE_FILEPATH",
nargs='?',
help="(Optional) Filepath to a NetCDF file containing"
" temperature lapse rates. If this cube is provided,"
" and a screen temperature cube is being processed,"
" the lapse rates will be used to adjust the"
" temperatures to better represent each spot's"
" site-altitude.")
parser.add_argument("output_filepath",
metavar="OUTPUT_FILEPATH",
help="The output path for the resulting NetCDF")
parser.add_argument(
"--apply_lapse_rate_correction",
default=False,
action="store_true",
help="If the option is set and a lapse rate cube has been "
"provided, extracted screen temperatures will be adjusted to "
"better match the altitude of the spot site for which they have "
"been extracted.")
method_group = parser.add_argument_group(
title="Neighbour finding method",
description="If none of these options are set, the nearest grid point "
"to a spot site will be used without any other constraints.")
method_group.add_argument(
"--land_constraint",
default=False,
action='store_true',
help="If set the neighbour cube will be interrogated for grid point"
" neighbours that were identified using a land constraint. This means"
" that the grid points should be land points except for sites where"
" none were found within the search radius when the neighbour cube was"
" created. May be used with minimum_dz.")
method_group.add_argument(
"--minimum_dz",
default=False,
action='store_true',
help="If set the neighbour cube will be interrogated for grid point"
" neighbours that were identified using a minimum height difference"
" constraint. These are grid points that were found to be the closest"
" in altitude to the spot site within the search radius defined when"
" the neighbour cube was created. May be used with land_constraint.")
percentile_group = parser.add_argument_group(
title="Extract percentiles",
description="Extract particular percentiles from probabilistic, "
"percentile, or realization inputs. If deterministic input is "
"provided a warning is raised and all leading dimensions are included "
"in the returned spot-data cube.")
percentile_group.add_argument(
"--extract_percentiles",
default=None,
nargs='+',
type=float,
help="If set to a percentile value or a list of percentile values, "
"data corresponding to those percentiles will be returned. For "
"example setting '--extract_percentiles 25 50 75' will result in the "
"25th, 50th, and 75th percentiles being returned from a cube of "
"probabilities, percentiles, or realizations. Note that for "
"percentile inputs, the desired percentile(s) must exist in the input "
"cube.")
parser.add_argument(
"--ecc_bounds_warning",
default=False,
action="store_true",
help="If True, where calculated percentiles are outside the ECC "
"bounds range, raise a warning rather than an exception.")
meta_group = parser.add_argument_group("Metadata")
meta_group.add_argument(
"--metadata_json",
metavar="METADATA_JSON",
default=None,
help="If provided, this JSON file can be used to modify the metadata "
"of the returned netCDF file. Defaults to None.")
output_group = parser.add_argument_group("Suppress Verbose output")
# This CLI may be used to prepare data for verification without knowing the
# form of the input, be it deterministic, realizations or probabilistic.
# A warning is normally raised when attempting to extract a percentile from
# deterministic data as this is not possible; the spot-extraction of the
# entire cube is returned. When preparing data for verification we know
# that we will produce a large number of these warnings when passing in
# deterministic data. This option to suppress warnings is provided to
# reduce the amount of unneeded logging information that is written out.
output_group.add_argument(
"--suppress_warnings",
default=False,
action="store_true",
help="Suppress warning output. This option should only be used if "
"it is known that warnings will be generated but they are not "
"required.")
args = parser.parse_args(args=argv)
# Load Cube and JSON.
neighbour_cube = load_cube(args.neighbour_filepath)
diagnostic_cube = load_cube(args.diagnostic_filepath)
lapse_rate_cube = load_cube(args.temperature_lapse_rate_filepath,
allow_none=True)
metadata_dict = load_json_or_none(args.metadata_json)
# Process Cube
result = process(neighbour_cube, diagnostic_cube, lapse_rate_cube,
args.apply_lapse_rate_correction, args.land_constraint,
args.minimum_dz, args.extract_percentiles,
args.ecc_bounds_warning, metadata_dict,
args.suppress_warnings)
# Save Cube
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Load in arguments and get going."""
parser = ArgParser(
description=('Reads input orography and landmask fields. Creates a '
'series of masks, where each mask excludes data below or'
' equal to the lower threshold, and excludes data above '
'the upper threshold.'))
parser.add_argument('input_filepath_standard_orography',
metavar='INPUT_FILE_STANDARD_OROGRAPHY',
help=('A path to an input NetCDF orography file to '
'be processed'))
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
parser.add_argument('--input_filepath_landmask',
metavar='INPUT_FILE_LAND',
help=('A path to an input NetCDF land mask file to be '
'processed. If provided, sea points will be '
'set to zero in every band. If '
'no land mask is provided, sea points will be '
'included in the appropriate topographic band.'))
parser.add_argument('--force',
dest='force',
default=False,
action='store_true',
help=('If keyword is set (i.e. True), ancillaries '
'will be generated even if doing so will '
'overwrite existing files'))
parser.add_argument('--thresholds_filepath',
metavar='THRESHOLDS_FILEPATH',
default=None,
help=("The path to a json file which can be used "
"to set the number and size of topographic "
"bounds. If unset a default bounds dictionary"
" will be used."
"The dictionary has the following form: "
"{'bounds': [[-500., 50.], [50., 100.], "
"[100., 150.],[150., 200.], [200., 250.], "
"[250., 300.], [300., 400.], [400., 500.], "
"[500., 650.],[650., 800.], [800., 950.], "
"[950., 6000.]], 'units': 'm'}"))
args = parser.parse_args(args=argv)
thresholds_dict = load_json_or_none(args.thresholds_filepath)
if thresholds_dict is None:
thresholds_dict = THRESHOLDS_DICT
if not os.path.exists(args.output_filepath) or args.force:
orography = load_cube(args.input_filepath_standard_orography)
landmask = None
if args.input_filepath_landmask:
try:
landmask = load_cube(args.input_filepath_landmask)
except IOError as err:
msg = ("Loading land mask has been unsuccessful: {}. "
"This may be because the land mask could not be "
"located in {}; run "
'improver-generate-landmask-ancillary first.').format(
err, args.input_filepath_landmask)
raise IOError(msg)
# Process Cube
result = process(orography, landmask, thresholds_dict)
# Save Cube
save_netcdf(result, args.output_filepath)
else:
print('File already exists here: ', args.output_filepath)
def main(argv=None):
"""Extrapolate data forward in time."""
parser = ArgParser(
description="Extrapolate input data to required lead times.")
parser.add_argument("input_filepath",
metavar="INPUT_FILEPATH",
type=str,
help="Path to input NetCDF file.")
group = parser.add_mutually_exclusive_group()
group.add_argument("--output_dir",
metavar="OUTPUT_DIR",
type=str,
default="",
help="Directory to write output files.")
group.add_argument("--output_filepaths",
nargs="+",
type=str,
help="List of full paths to output nowcast files, in "
"order of increasing lead time.")
optflw = parser.add_argument_group('Advect using files containing the x '
' and y components of the velocity')
optflw.add_argument("--eastward_advection_filepath",
type=str,
help="Path"
" to input file containing Eastward advection "
"velocities.")
optflw.add_argument("--northward_advection_filepath",
type=str,
help="Path"
" to input file containing Northward advection "
"velocities.")
speed = parser.add_argument_group('Advect using files containing speed and'
' direction')
speed.add_argument("--advection_speed_filepath",
type=str,
help="Path"
" to input file containing advection speeds,"
" usually wind speeds, on multiple pressure levels.")
speed.add_argument("--advection_direction_filepath",
type=str,
help="Path to input file containing the directions from"
" which advection speeds are coming (180 degrees from"
" the direction in which the speed is directed). The"
" directions should be on the same grid as the input"
" speeds, including the same vertical levels.")
speed.add_argument("--pressure_level",
type=int,
default=75000,
help="The"
" pressure level in Pa to extract from the multi-level"
" advection_speed and advection_direction files. The"
" velocities at this level are used for advection.")
parser.add_argument("--orographic_enhancement_filepaths",
nargs="+",
type=str,
default=None,
help="List or wildcarded "
"file specification to the input orographic "
"enhancement files. Orographic enhancement files are "
"compulsory for precipitation fields.")
parser.add_argument("--json_file",
metavar="JSON_FILE",
default=None,
help="Filename for the json file containing "
"required changes to the metadata. Information "
"describing the intended contents of the json file "
"is available in "
"improver.utilities.cube_metadata.amend_metadata."
"Every output cube will have the metadata_dict "
"applied. Defaults to None.",
type=str)
parser.add_argument("--max_lead_time",
type=int,
default=360,
help="Maximum lead time required (mins).")
parser.add_argument("--lead_time_interval",
type=int,
default=15,
help="Interval between required lead times (mins).")
accumulation_args = parser.add_argument_group(
'Calculate accumulations from advected fields')
accumulation_args.add_argument(
"--accumulation_fidelity",
type=int,
default=0,
help="If set, this CLI will additionally return accumulations"
" calculated from the advected fields. This fidelity specifies the"
" time interval in minutes between advected fields that is used to"
" calculate these accumulations. This interval must be a factor of"
" the lead_time_interval.")
accumulation_args.add_argument(
"--accumulation_period",
type=int,
default=15,
help="The period over which the accumulation is calculated (mins). "
"Only full accumulation periods will be computed. At lead times "
"that are shorter than the accumulation period, no accumulation "
"output will be produced.")
accumulation_args.add_argument(
"--accumulation_units",
type=str,
default='m',
help="Desired units in which the accumulations should be expressed,"
"e.g. mm")
# Load Cubes
args = parser.parse_args(args=argv)
metadata_dict = load_json_or_none(args.json_file)
upath, vpath = (args.eastward_advection_filepath,
args.northward_advection_filepath)
spath, dpath = (args.advection_speed_filepath,
args.advection_direction_filepath)
# load files and initialise advection plugin
input_cube = load_cube(args.input_filepath)
orographic_enhancement_cube = load_cube(
args.orographic_enhancement_filepaths, allow_none=True)
speed_cube = direction_cube = ucube = vcube = None
if (upath and vpath) and not (spath or dpath):
ucube = load_cube(upath)
vcube = load_cube(vpath)
elif (spath and dpath) and not (upath or vpath):
level_constraint = Constraint(pressure=args.pressure_level)
try:
speed_cube = load_cube(spath, constraints=level_constraint)
direction_cube = load_cube(dpath, constraints=level_constraint)
except ValueError as err:
raise ValueError(
'{} Unable to extract specified pressure level from given '
'speed and direction files.'.format(err))
else:
raise ValueError('Cannot mix advection component velocities with speed'
' and direction')
# Process Cubes
accumulation_cubes, forecast_to_return = process(
input_cube, ucube, vcube, speed_cube, direction_cube,
orographic_enhancement_cube, metadata_dict, args.max_lead_time,
args.lead_time_interval, args.accumulation_fidelity,
args.accumulation_period, args.accumulation_units)
# Save Cube
if args.output_filepaths and \
len(args.output_filepaths) != len(forecast_to_return):
raise ValueError("Require exactly one output file name for each "
"forecast lead time")
for i, cube in enumerate(forecast_to_return):
# save to a suitably-named output file
if args.output_filepaths:
file_name = args.output_filepaths[i]
else:
file_name = os.path.join(args.output_dir, generate_file_name(cube))
save_netcdf(cube, file_name)
if args.accumulation_fidelity > 0:
# return accumulation cubes
for i, cube in enumerate(accumulation_cubes):
file_name = os.path.join(args.output_dir, generate_file_name(cube))
save_netcdf(cube, file_name)
def main(argv=None):
"""Load in arguments and get going."""
description = (
"Determine grid point coordinates within the provided cubes that "
"neighbour spot data sites defined within the provided JSON "
"file. If no options are set the returned netCDF file will contain the"
" nearest neighbour found for each site. Other constrained neighbour "
"finding methods can be set with options below.")
options = ("\n\nThese methods are:\n\n"
" 1. nearest neighbour\n"
" 2. nearest land point neighbour\n"
" 3. nearest neighbour with minimum height difference\n"
" 4. nearest land point neighbour with minimum height "
"difference")
parser = ArgParser(
description=('\n'.join(wrap(description, width=79)) + options),
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument("site_list_filepath", metavar="SITE_LIST_FILEPATH",
help="Path to a JSON file that contains the spot sites"
" for which neighbouring grid points are to be found.")
parser.add_argument("orography_filepath", metavar="OROGRAPHY_FILEPATH",
help="Path to a NetCDF file of model orography for the"
" model grid on which neighbours are being found.")
parser.add_argument("landmask_filepath", metavar="LANDMASK_FILEPATH",
help="Path to a NetCDF file of model land mask for the"
" model grid on which neighbours are being found.")
parser.add_argument("output_filepath", metavar="OUTPUT_FILEPATH",
help="The output path for the resulting NetCDF")
parser.add_argument(
"--all_methods", default=False, action='store_true',
help="If set this will return a cube containing the nearest grid point"
" neighbours to spot sites as defined by each possible combination of"
" constraints.")
group = parser.add_argument_group('Apply constraints to neighbour choice')
group.add_argument(
"--land_constraint", default=False, action='store_true',
help="If set this will return a cube containing the nearest grid point"
" neighbours to spot sites that are also land points. May be used with"
" the minimum_dz option.")
group.add_argument(
"--minimum_dz", default=False, action='store_true',
help="If set this will return a cube containing the nearest grid point"
" neighbour to each spot site that is found, within a given search"
" radius, to minimise the height difference between the two. May be"
" used with the land_constraint option.")
group.add_argument(
"--search_radius", metavar="SEARCH_RADIUS", type=float,
help="The radius in metres about a spot site within which to search"
" for a grid point neighbour that is land or which has a smaller "
" height difference than the nearest. The default value is 10000m "
"(10km).")
group.add_argument(
"--node_limit", metavar="NODE_LIMIT", type=int,
help="When searching within the defined search_radius for suitable "
"neighbours, a KDTree is constructed. This node_limit prevents the "
"tree from becoming too large for large search radii. A default of 36"
" is set, which is to say the nearest 36 grid points will be "
"considered. If the search_radius is likely to contain more than 36 "
"points, this value should be increased to ensure all points are "
"considered.")
s_group = parser.add_argument_group('Site list options')
s_group.add_argument(
"--site_coordinate_system", metavar="SITE_COORDINATE_SYSTEM",
help="The coordinate system in which the site coordinates are provided"
" within the site list. This must be provided as the name of a cartopy"
" coordinate system. The default is a PlateCarree system, with site"
" coordinates given by latitude/longitude pairs. This can be a"
" complete definition, including parameters required to modify a"
" default system, e.g. Miller(central_longitude=90). If a globe is"
" required this can be specified as e.g."
" Globe(semimajor_axis=100, semiminor_axis=100).")
s_group.add_argument(
"--site_coordinate_options", metavar="SITE_COORDINATE_OPTIONS",
help="JSON formatted string of options passed to the cartopy"
" coordinate system given in site_coordinate_system. \"globe\""
" is handled as a special case for options to construct a cartopy"
" Globe object.")
s_group.add_argument(
"--site_x_coordinate", metavar="SITE_X_COORDINATE",
help="The x coordinate key within the JSON file. The plugin default is"
" 'longitude', but can be changed using this option if required.")
s_group.add_argument(
"--site_y_coordinate", metavar="SITE_Y_COORDINATE",
help="The y coordinate key within the JSON file. The plugin default is"
" 'latitude', but can be changed using this option if required.")
args = parser.parse_args(args=argv)
# Load Cubes and JSON.
site_list = load_json_or_none(args.site_list_filepath)
orography = load_cube(args.orography_filepath)
landmask = load_cube(args.landmask_filepath)
# Process Cube
result = process(orography, landmask, site_list,
args.all_methods, args.land_constraint, args.minimum_dz,
args.search_radius, args.node_limit,
args.site_coordinate_system, args.site_coordinate_options,
args.site_x_coordinate, args.site_y_coordinate)
# Save Cube
save_netcdf(result, args.output_filepath)
