def test_visualization_plot_precip_field(
source, type, bbox, colorscale, probthr, title, colorbar, axis,
):
if type == "intensity":
field, metadata = get_precipitation_fields(0, 0, True, True, None, source)
field = field.squeeze()
field, metadata = conversion.to_rainrate(field, metadata)
elif type == "depth":
field, metadata = get_precipitation_fields(0, 0, True, True, None, source)
field = field.squeeze()
field, metadata = conversion.to_raindepth(field, metadata)
elif type == "prob":
field, metadata = get_precipitation_fields(0, 10, True, True, None, source)
field, metadata = conversion.to_rainrate(field, metadata)
field = ensemblestats.excprob(field, probthr)
ax = plot_precip_field(
field,
type=type,
bbox=bbox,
geodata=metadata,
colorscale=colorscale,
probthr=probthr,
units=metadata["unit"],
title=title,
colorbar=colorbar,
axis=axis,
)
pl.close()
def test_visualization_plot_precip_field(
source,
plot_type,
bbox,
colorscale,
probthr,
title,
colorbar,
axis,
):
if plot_type == "intensity":
field, metadata = get_precipitation_fields(0, 0, True, True, None,
source)
field = field.squeeze()
field, metadata = conversion.to_rainrate(field, metadata)
elif plot_type == "depth":
field, metadata = get_precipitation_fields(0, 0, True, True, None,
source)
field = field.squeeze()
field, metadata = conversion.to_raindepth(field, metadata)
elif plot_type == "prob":
field, metadata = get_precipitation_fields(0, 10, True, True, None,
source)
field, metadata = conversion.to_rainrate(field, metadata)
field = ensemblestats.excprob(field, probthr)
field_orig = field.copy()
ax = plot_precip_field(
field.copy(),
type=plot_type,
bbox=bbox,
geodata=None,
colorscale=colorscale,
probthr=probthr,
units=metadata["unit"],
title=title,
colorbar=colorbar,
axis=axis,
)
# Check that plot_precip_field does not modify the input data
field_orig = np.ma.masked_invalid(field_orig)
field_orig.data[field_orig.mask] = -100
field = np.ma.masked_invalid(field)
field.data[field.mask] = -100
assert np.array_equal(field_orig.data, field.data)
source = "".join([i for i in case if not i.isdigit()])
data_source = pysteps.rcparams.data_sources[source]
# Find the input files from the archive
file_names = io.archive.find_by_date(
case_date,
data_source["root_path"],
data_source["path_fmt"],
data_source["fn_pattern"],
data_source["fn_ext"],
data_source["timestep"],
num_prev_files=0,
num_next_files=frames - 1,
)
if None in file_names[0]:
raise FileNotFoundError(
f"Error loading {case} case. Some files are missing.")
# Read the radar composites
importer = io.get_method(data_source["importer"], "importer")
importer_kwargs = data_source["importer_kwargs"]
reflectivity, _, metadata = io.read_timeseries(file_names, importer,
**importer_kwargs)
# Convert to rain rate
precip, metadata = conversion.to_rainrate(reflectivity, metadata)
return precip, metadata, data_source["timestep"]
fn_ext,
timestep=5,
num_prev_files=1)
# Read the radar composites
importer = io.get_method(importer_name, "importer")
R, quality, metadata = io.read_timeseries(fns, importer, **importer_kwargs)
del quality # Not used
###############################################################################
# Preprocess the data
# ~~~~~~~~~~~~~~~~~~~
# Convert to mm/h
R, metadata = conversion.to_rainrate(R, metadata)
# Keep the reference frame in mm/h and its mask (for plotting purposes)
ref_mm = R[0, :, :].copy()
mask = np.ones(ref_mm.shape)
mask[~np.isnan(ref_mm)] = np.nan
# Log-transform the data [dBR]
R, metadata = transformation.dB_transform(R,
metadata,
threshold=0.1,
zerovalue=-15.0)
# Keep the reference frame in dBR (for plotting purposes)
ref_dbr = R[0].copy()
ref_dbr[ref_dbr < -10] = np.nan
path_fmt,
fn_pattern,
fn_ext,
timestep,
num_next_files=11,
)
# Read the radar composites
importer = io.get_method(importer_name, "importer")
Z, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs)
# Keep only positive rainfall values
Z = Z[Z > metadata["zerovalue"]].flatten()
# Convert to rain rate
R, metadata = conversion.to_rainrate(Z, metadata)
###############################################################################
# Test data transformations
# -------------------------
# Define method to visualize the data distribution with boxplots and plot the
# corresponding skewness
def plot_distribution(data, labels, skw):
N = len(data)
fig, ax1 = plt.subplots()
ax2 = ax1.twinx()
ax2.plot(np.arange(N + 2), np.zeros(N + 2), ":r")
ax1.boxplot(data, labels=labels, sym="", medianprops={"color": "k"})
low_quality = False
for timestamp in metadata["timestamps"]:
if timestamp.strftime("%Y%m%d%H%M") in badts:
print(
"Skipping, low quality observation found for time step %d"
% (i + 1))
low_quality = True
break
if missing_data or low_quality:
curdate += timedelta(minutes=timestep)
continue
R[~np.isfinite(R)] = metadata["zerovalue"]
R, metadata = conversion.to_rainrate(R, metadata)
R, metadata = transformation.dB_transform(R, metadata, threshold=R_min)
obs_fns = io.archive.find_by_date(curdate,
root_path,
datasource["path_fmt"],
datasource["fn_pattern"],
datasource["fn_ext"],
datasource["timestep"],
num_next_files=num_timesteps)
obs_fns = (obs_fns[0][1:], obs_fns[1][1:])
if len(obs_fns[0]) == 0:
curdate += timedelta(minutes=timestep)
print("Skipping, no verifying observations found.")
continue
def test_to_rainrate(R, metadata, expected):
"""Test the to_rainrate."""
assert_array_almost_equal(conversion.to_rainrate(R, metadata)[0], expected)
# Find the input files from the archive
fns = io.archive.find_by_date(date,
root_path,
path_fmt,
fn_pattern,
fn_ext,
timestep,
num_prev_files=2)
# Read the radar composites
importer = io.get_method(importer_name, "importer")
Z, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs)
# Convert to rain rate using the finnish Z-R relationship
R, metadata = conversion.to_rainrate(Z, metadata, 223.0, 1.53)
# Plot the rainfall field
plot_precip_field(R[-1, :, :], geodata=metadata)
# Store the last frame for plotting it later later
R_ = R[-1, :, :].copy()
# Log-transform the data to unit of dBR, set the threshold to 0.1 mm/h,
# set the fill value to -15 dBR
R, metadata = transformation.dB_transform(R,
metadata,
threshold=0.1,
zerovalue=-15.0)
# Nicely print the metadata
while curdate <= enddate:
print("Computing nowcasts for event %d, start date %s..." % (pei+1, str(curdate)), end="")
sys.stdout.flush()
if curdate + num_timesteps * timedelta(minutes=5) > enddate:
break
fns = io.archive.find_by_date(curdate, root_path, datasource["path_fmt"],
datasource["fn_pattern"], datasource["fn_ext"],
datasource["timestep"], num_prev_files=9)
R,_,metadata = io.readers.read_timeseries(fns, importer,
**datasource["importer_kwargs"])
if domain == "fmi":
R, metadata = conversion.to_rainrate(R, metadata, a=223.0, b=1.53)
if upscale_factor > 1:
R_ = []
for i in range(R.shape[0]):
R_.append(upscale_precip_field(R[i, :, :], upscale_factor))
R = np.stack(R_)
missing_data = False
for i in range(R.shape[0]):
if not np.any(np.isfinite(R[i, :, :])):
print("Skipping, no finite values found for time step %d" % (i+1))
missing_data = True
break
if missing_data:
curdate += timedelta(minutes=timestep)
fn_pattern,
fn_ext,
timestep,
num_next_files=11,
)
# Read the radar composites
importer = io.get_method(importer_name, "importer")
Z, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs)
# Keep only positive rainfall values
Z = Z[Z > metadata["zerovalue"]].flatten()
# Convert to rain rate using the finnish Z-R relationship
# Z = 223*R^1.53
R, metadata = conversion.to_rainrate(Z, metadata, 223.0, 1.53)
###############################################################################
# Test data transformations
# -------------------------
# Define method to visualize the data distribution with boxplots and plot the
# corresponding skewness
def plot_distribution(data, labels, skw):
N = len(data)
fig, ax1 = plt.subplots()
ax2 = ax1.twinx()
ax2.plot(np.arange(N + 2), np.zeros(N + 2), ":r")
# find the files containing the verifying observations
fns = io.archive.find_by_date(date,
root_path,
"%Y%m%d",
fn_pattern,
fn_ext,
5,
0,
num_next_files=12)
# read the observations
Z_obs, _, metadata = io.read_timeseries(fns,
import_fmi_pgm,
gzipped=True,
num_next_files=12)
R_obs = conversion.to_rainrate(Z_obs, metadata, 223.0, 1.53)[0]
# iterate over the nowcast lead times
for lt in range(R_f.shape[1]):
# compute the exceedance probability of 0.1 mm/h from the ensemble
P_f = ensemblestats.excprob(R_f[:, lt, :, :], 0.1, ignore_nan=True)
# compute and plot the ROC curve
roc = verification.ROC_curve_init(0.1, n_prob_thrs=10)
verification.ROC_curve_accum(roc, P_f, R_obs[lt + 1, :, :])
fig = figure()
verification.plot_ROC(roc, ax=fig.gca(), opt_prob_thr=True)
pyplot.savefig("ROC_%02d.eps" % (lt + 1), bbox_inches="tight")
pyplot.close()
# compute and plot the reliability diagram
def forecast(
precip,
precip_metadata,
velocity,
timesteps,
timestep,
nowcast_method,
precip_nwp=None,
precip_nwp_metadata=None,
start_blending=120,
end_blending=240,
fill_nwp=True,
nowcast_kwargs=None,
):
"""Generate a forecast by linearly blending nowcasts with NWP data
Parameters
----------
precip: array_like
Array containing the input precipitation field(s) ordered by timestamp
from oldest to newest. The time steps between the inputs are assumed
to be regular.
precip_metadata: dict
Metadata dictionary containing (at least) the transform, unit and threshold
attributes as described in the documentation of :py:mod:`pysteps.io.importers`.
velocity; array_like
Array of shape (2, m, n) containing the x- and y-components of the advection
field. The velocities are assumed to represent one time step between the
inputs. All values are required to be finite.
timesteps: int
Number of time steps to forecast.
timestep: int or float
The time difference (in minutes) between consecutive forecast fields.
nowcast_method: str
Name of the nowcasting method. See :py:mod:`pysteps.nowcasts.interface`
for the list of available methods.
precip_nwp: array_like or NoneType, optional
Array of shape (timesteps, m, n) in the case of no ensemble or
of shape (n_ens_members, timesteps, m, n) in the case of an ensemble
containing the NWP precipitation fields ordered by timestamp from oldest
to newest. The time steps between the inputs are assumed to be regular
(and identical to the time step between the nowcasts). If no NWP
data is given the value of precip_nwp is None and no blending will be performed.
precip_nwp_metadata: dict or NoneType, optional
NWP metadata dictionary containing (at least) the transform, unit and threshold
attributes as described in the documentation of :py:mod:`pysteps.io.importers`.
start_blending: int, optional
Time stamp (in minutes) after which the blending should start. Before this
only the nowcast data is used.
end_blending: int, optional
Time stamp (in minutes) after which the blending should end. Between
start_blending and end_blending the nowcasts and NWP data are linearly
merged with each other. After end_blending only the NWP data is used.
fill_nwp: bool, optional
Standard value is True. If True, the NWP data will be used to fill in the
no data mask of the nowcast.
nowcast_kwargs: dict, optional
Dictionary containing keyword arguments for the nowcast method.
Returns
-------
R_blended: ndarray
Array of shape (timesteps, m, n) in the case of no ensemble or
of shape (n_ens_members, timesteps, m, n) in the case of an ensemble
containing the precipation forecast generated by linearly blending
the nowcasts and the NWP data. n_ens_members equals the maximum no. of
ensemble members in either the nowcast or nwp model(s).
"""
if nowcast_kwargs is None:
nowcast_kwargs = dict()
# Calculate the nowcasts
nowcast_method_func = nowcasts.get_method(nowcast_method)
precip_nowcast = nowcast_method_func(
precip,
velocity,
timesteps,
**nowcast_kwargs,
)
# Make sure that precip_nowcast and precip_nwp are in mm/h
precip_nowcast, _ = conversion.to_rainrate(precip_nowcast, metadata=precip_metadata)
# Check if NWP data is given as input
if precip_nwp is not None:
precip_nwp, _ = conversion.to_rainrate(precip_nwp, metadata=precip_nwp_metadata)
if len(precip_nowcast.shape) == 4:
n_ens_members_nowcast = precip_nowcast.shape[0]
if n_ens_members_nowcast == 1:
precip_nowcast = np.squeeze(precip_nowcast)
else:
n_ens_members_nowcast = 1
if len(precip_nwp.shape) == 4:
n_ens_members_nwp = precip_nwp.shape[0]
if n_ens_members_nwp == 1:
precip_nwp = np.squeeze(precip_nwp)
else:
n_ens_members_nwp = 1
# Now, repeat the nowcast ensemble members or the nwp models/members until
# it has the same amount of members as n_ens_members_max. For instance, if
# you have 10 ensemble nowcasts members and 3 NWP members, the output will
# be an ensemble of 10 members. Hence, the three NWP members are blended
# with the first three members of the nowcast (member one with member one,
# two with two, etc.), subsequently, the same NWP members are blended with
# the next three members (NWP member one with member 4, NWP member 2 with
# member 5, etc.), until 10 is reached.
n_ens_members_max = max(n_ens_members_nowcast, n_ens_members_nwp)
n_ens_members_min = min(n_ens_members_nowcast, n_ens_members_nwp)
if n_ens_members_min != n_ens_members_max:
if n_ens_members_nwp == 1:
precip_nwp = np.repeat(
precip_nwp[np.newaxis, :, :], n_ens_members_max, axis=0
)
elif n_ens_members_nowcast == 1:
precip_nowcast = np.repeat(
precip_nowcast[np.newaxis, :, :], n_ens_members_max, axis=0
)
else:
repeats = [
(n_ens_members_max + i) // n_ens_members_min
for i in range(n_ens_members_min)
]
if n_ens_members_nwp == n_ens_members_min:
precip_nwp = np.repeat(precip_nwp, repeats, axis=0)
elif n_ens_members_nowcast == n_ens_members_min:
precip_nowcast = np.repeat(precip_nowcast, repeats, axis=0)
# Check if dimensions are correct
assert (
precip_nwp.shape == precip_nowcast.shape
), "The dimensions of precip_nowcast and precip_nwp need to be identical: dimension of precip_nwp = {} and dimension of precip_nowcast = {}".format(
precip_nwp.shape, precip_nowcast.shape
)
# Initialise output
R_blended = np.zeros_like(precip_nowcast)
# Calculate the weights
for i in range(timesteps):
# Calculate what time we are at
t = (i + 1) * timestep
# Calculate the weight with a linear relation (weight_nwp at start_blending = 0.0)
# and (weight_nwp at end_blending = 1.0)
weight_nwp = (t - start_blending) / (end_blending - start_blending)
# Set weights at times before start_blending and after end_blending
if weight_nwp < 0.0:
weight_nwp = 0.0
elif weight_nwp > 1.0:
weight_nwp = 1.0
# Calculate weight_nowcast
weight_nowcast = 1.0 - weight_nwp
# Calculate output by combining precip_nwp and precip_nowcast,
# while distinguishing between ensemble and non-ensemble methods
if n_ens_members_max == 1:
R_blended[i, :, :] = (
weight_nwp * precip_nwp[i, :, :]
+ weight_nowcast * precip_nowcast[i, :, :]
)
else:
R_blended[:, i, :, :] = (
weight_nwp * precip_nwp[:, i, :, :]
+ weight_nowcast * precip_nowcast[:, i, :, :]
)
# Find where the NaN values are and replace them with NWP data
if fill_nwp:
nan_indices = np.isnan(R_blended)
R_blended[nan_indices] = precip_nwp[nan_indices]
else:
# If no NWP data is given, the blended field is simply equal to the nowcast field
R_blended = precip_nowcast
return R_blended
