def S_PROG(self,
history,
observation,
th_dbz,
n_leadtimes,
n_frames=3,
block_sizes=7,
win_sizes=8,
decl_scales=3,
n_cascade_levels=3):
V = self.getV(history, n_frames, block_sizes, win_sizes, decl_scales)
th = th_dbz / 70 * 255
sprog = nowcasts.get_method("sprog")
prediction_sprog = sprog(
history, # it takes last three
V,
n_leadtimes,
n_cascade_levels=n_cascade_levels, # 3-4
R_thr=th, # reflectivity th
decomp_method="fft",
bandpass_filter_method="gaussian",
probmatching_method="mean",
)
return prediction_sprog
def test_steps(
n_ens_members,
n_cascade_levels,
ar_order,
mask_method,
probmatching_method,
max_crps,
):
"""Tests STEPS nowcast."""
# inputs
R, metadata = _import_mch_gif(2, 0)
R_o = _import_mch_gif(0, 3)[0][1:, :, :]
# optical flow
of_method = motion.get_method("LK")
V = of_method(R)
# nowcast
nowcast_method = nowcasts.get_method("steps")
num_timesteps = 1
R_f = nowcast_method(
R,
V,
n_timesteps=3,
R_thr=metadata["threshold"],
kmperpixel=2.0,
timestep=metadata["accutime"],
seed=42,
n_ens_members=n_ens_members,
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
mask_method=mask_method,
probmatching_method=probmatching_method,
)
# result
result = verification.probscores.CRPS(R_f[-1], R_o[-1])
assert result < max_crps
def test_steps_skill(
n_ens_members,
n_cascade_levels,
ar_order,
mask_method,
probmatching_method,
domain,
timesteps,
max_crps,
):
"""Tests STEPS nowcast skill."""
# inputs
precip_input, metadata = get_precipitation_fields(
num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(
num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000
)[1:, :, :]
precip_obs = precip_obs.filled()
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
nowcast_method = nowcasts.get_method("steps")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
timesteps=timesteps,
R_thr=metadata["threshold"],
kmperpixel=2.0,
timestep=metadata["accutime"],
seed=42,
n_ens_members=n_ens_members,
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
mask_method=mask_method,
probmatching_method=probmatching_method,
domain=domain,
)
assert precip_forecast.ndim == 4
assert precip_forecast.shape[0] == n_ens_members
assert precip_forecast.shape[1] == (
timesteps if isinstance(timesteps, int) else len(timesteps)
)
crps = verification.probscores.CRPS(precip_forecast[:, -1], precip_obs[-1])
assert crps < max_crps, f"CRPS={crps:.2f}, required < {max_crps:.2f}"
def test_steps(
n_ens_members,
n_cascade_levels,
ar_order,
mask_method,
probmatching_method,
domain,
max_crps,
):
"""Tests STEPS nowcast."""
# inputs
precip_input, metadata = get_precipitation_fields(
num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(num_prev_files=0,
num_next_files=3,
return_raw=False,
upscale=2000)[1:, :, :]
precip_obs = precip_obs.filled()
# Retrieve motion field
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
# Run nowcast
nowcast_method = nowcasts.get_method("steps")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
n_timesteps=3,
R_thr=metadata["threshold"],
kmperpixel=2.0,
timestep=metadata["accutime"],
seed=42,
n_ens_members=n_ens_members,
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
mask_method=mask_method,
probmatching_method=probmatching_method,
domain=domain,
)
# result
crps = verification.probscores.CRPS(precip_forecast[-1], precip_obs[-1])
print(f"got CRPS={crps:.1f}, required < {max_crps:.1f}")
assert crps < max_crps
def extrapolation(self,
history,
observation,
th_dbz,
n_leadtimes,
n_frames=3,
block_sizes=4,
win_sizes=8,
decl_scales=2):
V = self.getV(history, n_frames, block_sizes, win_sizes, decl_scales)
extrapolate = nowcasts.get_method("extrapolation")
prediction_extrapolate = extrapolate(history[-1], V, n_leadtimes)
return prediction_extrapolate
def test_sprog(
n_cascade_levels,
ar_order,
probmatching_method,
domain,
min_csi):
"""Tests SPROG nowcast."""
# inputs
precip_input, metadata = get_precipitation_fields(num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(num_prev_files=0,
num_next_files=3,
return_raw=False,
upscale=2000)[1:, :, :]
precip_obs = precip_obs.filled()
# Retrieve motion field
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
# Run nowcast
nowcast_method = nowcasts.get_method("sprog")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
n_timesteps=3,
R_thr=metadata["threshold"],
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
probmatching_method=probmatching_method,
domain=domain,
)
# result
result = verification.det_cat_fct(
precip_forecast[-1],
precip_obs[-1],
thr=0.1,
scores="CSI")["CSI"]
print(f"got CSI={result:.1f}, required > {min_csi:.1f}")
assert result > min_csi
def STEPS(self,
history,
observation,
th_dbz,
n_leadtimes,
n_frames=4,
block_sizes=6,
win_sizes=6,
decl_scales=3,
n_cascade_levels=3,
ar_window_radius=None):
V = self.getV(history, n_frames, block_sizes, win_sizes, decl_scales)
if observation is not None:
n_ens_members = 20
else:
n_ens_members = 100
th = th_dbz / 70 * 255
steps = nowcasts.get_method("steps")
prediction_steps = steps(
history, # last 3
V,
n_leadtimes,
n_ens_members=20,
n_cascade_levels=n_cascade_levels, # 3-4
R_thr=th, # Z th
kmperpixel=4.6875, # pixel resolution in km 64/300=4.6875
timestep=5,
decomp_method="fft",
bandpass_filter_method="gaussian",
noise_method="nonparametric",
vel_pert_method="bps",
mask_method="incremental",
seed=24,
)
#Two options: get the best prediction or do the average (for the avg use 50 or 100 samples)
if observation is not None:
best_ETS = -2
for p in prediction_steps:
ETS = self.getETS(observation, p, th_dbz=th_dbz)
if ETS > best_ETS:
best_ETS = ETS
best_prediction = p
else:
best_prediction = prediction_steps.mean(axis=0)
return best_prediction
def DARTS(self,
history,
observation,
th_dbz,
n_leadtimes,
n_frames=3,
block_sizes=4,
win_sizes=8,
decl_scales=2):
oflow_method = motion.get_method("DARTS")
V = oflow_method(history)
extrapolate = nowcasts.get_method("extrapolation")
prediction_extrapolate = extrapolate(history[-1], V, n_leadtimes)
return prediction_extrapolate
def test_sprog(
n_cascade_levels, ar_order, probmatching_method, domain, timesteps, min_csi
):
"""Tests SPROG nowcast."""
# inputs
precip_input, metadata = get_precipitation_fields(
num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(
num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000
)[1:, :, :]
precip_obs = precip_obs.filled()
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
nowcast_method = nowcasts.get_method("sprog")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
timesteps=timesteps,
R_thr=metadata["threshold"],
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
probmatching_method=probmatching_method,
domain=domain,
)
assert precip_forecast.ndim == 3
assert precip_forecast.shape[0] == (
timesteps if isinstance(timesteps, int) else len(timesteps)
)
result = verification.det_cat_fct(
precip_forecast[-1], precip_obs[-1], thr=0.1, scores="CSI"
)["CSI"]
assert result > min_csi, f"CSI={result:.1f}, required > {min_csi:.1f}"
def test_steps(
n_ens_members,
n_cascade_levels,
ar_order,
mask_method,
probmatching_method,
max_crps):
"""Tests STEPS nowcast."""
# inputs
precip_input, metadata = get_precipitation_fields(num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000)
precip_obs = get_precipitation_fields(num_prev_files=0,
num_next_files=3,
return_raw=False,
upscale=2000)[1:, :, :]
# Retrieve motion field
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
# Run nowcast
nowcast_method = nowcasts.get_method("steps")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
n_timesteps=3,
R_thr=metadata["threshold"],
kmperpixel=2.0,
timestep=metadata["accutime"],
seed=42,
n_ens_members=n_ens_members,
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
mask_method=mask_method,
probmatching_method=probmatching_method,
)
# result
result = verification.probscores.CRPS(precip_forecast[-1], precip_obs[-1])
assert result < max_crps
def test_anvil_rainrate(n_cascade_levels, ar_order, ar_window_radius,
timesteps, min_csi):
"""Tests ANVIL nowcast using rain rate precipitation fields."""
# inputs
precip_input, metadata = get_precipitation_fields(
num_prev_files=4,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(num_prev_files=0,
num_next_files=3,
return_raw=False,
upscale=2000)[1:, :, :]
precip_obs = precip_obs.filled()
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
nowcast_method = nowcasts.get_method("anvil")
precip_forecast = nowcast_method(
precip_input[-(ar_order + 2):],
retrieved_motion,
timesteps=timesteps,
rainrate=None, # no R(VIL) conversion is done
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
ar_window_radius=ar_window_radius,
)
assert precip_forecast.ndim == 3
assert precip_forecast.shape[0] == (timesteps if isinstance(
timesteps, int) else len(timesteps))
result = verification.det_cat_fct(precip_forecast[-1],
precip_obs[-1],
thr=0.1,
scores="CSI")["CSI"]
assert result > min_csi, f"CSI={result:.2f}, required > {min_csi:.2f}"
def t_interp(t_boundary, img_boundary, analysis_time):
"""interpolate the two images temporally"""
tstart2t0_weight, t02tend_weight = \
get_t_weights(analysis_time, t_boundary)
dense_lucaskanade = motion.get_method('LK')
uv_forward = dense_lucaskanade(img_boundary)
uv_backward = dense_lucaskanade(img_boundary[::-1, :, :])
uv_forward, uv_backward = get_mean_uv(uv_forward, uv_backward)
extrapolate = nowcasts.get_method('extrapolation')
r_from_tstart = extrapolate(img_boundary[0, :, :],
uv_forward * t02tend_weight, 1)[0, :, :]
r_from_tend = extrapolate(img_boundary[1, :, :],
uv_backward * tstart2t0_weight, 1)[0, :, :]
r = r_from_tstart*tstart2t0_weight + \
r_from_tend*t02tend_weight
return r
def ANVIL(self,
history,
observation,
th_dbz,
n_leadtimes,
n_frames=3,
block_sizes=6,
win_sizes=9,
decl_scales=3,
n_cascade_levels=23,
ar_window_radius=3):
V = self.getV(history, n_frames, block_sizes, win_sizes, decl_scales)
anvil = nowcasts.get_method("anvil")
prediction_anvil = anvil(
history[-3:],
V,
n_leadtimes,
n_cascade_levels=n_cascade_levels,
ar_window_radius=ar_window_radius, # half of domain 15-40
ar_order=1)
return prediction_anvil
################################################################################
# Notice how the presence of erroneous velocity vectors produces a significantly
# slower motion field near the right edge of the domain.
#
# Forecast skill
# --------------
#
# We are now going to evaluate the benefit of buffering the radar mask by computing
# the forecast skill in terms of the Spearman correlation coefficient.
# The extrapolation forecasts are computed using the dense UV motion fields
# estimated above.
# Get the advection routine and extrapolate the last radar frame by 12 time steps
# (i.e., 1 hour lead time)
extrapolate = nowcasts.get_method("extrapolation")
R[~np.isfinite(R)] = metadata["zerovalue"]
R_f1 = extrapolate(R[-1], UV1, 12)
R_f2 = extrapolate(R[-1], UV2, 12)
# Back-transform to rain rate
R_f1 = transformation.dB_transform(R_f1, threshold=-10.0, inverse=True)[0]
R_f2 = transformation.dB_transform(R_f2, threshold=-10.0, inverse=True)[0]
# Find the veriyfing observations in the archive
fns = io.archive.find_by_date(date,
root_path,
path_fmt,
fn_pattern,
fn_ext,
timestep=5,
# Listing 2
from matplotlib import pyplot
from pysteps import motion, nowcasts
from pysteps.postprocessing.ensemblestats import excprob
from pysteps.utils import transformation
from pysteps.visualization import plot_precip_field
# compute the advection field
oflow_method = motion.get_method("lucaskanade")
V = oflow_method(R)
# compute the S-PROG nowcast
nowcast_method = nowcasts.get_method("sprog")
R_f_sprog = nowcast_method(R[-3:, :, :], V, 12, R_thr=-10.0)[-1, :, :]
# compute the STEPS nowcast
nowcast_method = nowcasts.get_method("steps")
R_f = nowcast_method(R[-3:, :, :],
V,
12,
n_ens_members=24,
n_cascade_levels=8,
R_thr=-10.0,
kmperpixel=1.0,
timestep=5)
# plot the S-PROG nowcast, one ensemble member of the STEPS nowcast and the exceedance
# probability of 0.1 mm/h computed from the ensemble
R_f_sprog = transformation.dB_transform(R_f_sprog,
threshold=-10.0,
def test_steps_callback(tmp_path):
"""Test STEPS callback functionality to export the output as a netcdf."""
n_ens_members = 2
n_timesteps = 3
precip_input, metadata = get_precipitation_fields(
num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip_input = precip_input.filled()
field_shape = (precip_input.shape[1], precip_input.shape[2])
startdate = metadata["timestamps"][-1]
timestep = metadata["accutime"]
motion_field = np.zeros((2, *field_shape))
exporter = io.initialize_forecast_exporter_netcdf(
outpath=tmp_path.as_posix(),
outfnprefix="test_steps",
startdate=startdate,
timestep=timestep,
n_timesteps=n_timesteps,
shape=field_shape,
n_ens_members=n_ens_members,
metadata=metadata,
incremental="timestep",
)
def callback(array):
return io.export_forecast_dataset(array, exporter)
precip_output = nowcasts.get_method("steps")(
precip_input,
motion_field,
timesteps=n_timesteps,
R_thr=metadata["threshold"],
kmperpixel=2.0,
timestep=timestep,
seed=42,
n_ens_members=n_ens_members,
vel_pert_method=None,
callback=callback,
return_output=True,
)
io.close_forecast_files(exporter)
# assert that netcdf exists and its size is not zero
tmp_file = os.path.join(tmp_path, "test_steps.nc")
assert os.path.exists(tmp_file) and os.path.getsize(tmp_file) > 0
# assert that the file can be read by the nowcast importer
precip_netcdf, metadata_netcdf = io.import_netcdf_pysteps(tmp_file, dtype="float64")
# assert that the dimensionality of the array is as expected
assert precip_netcdf.ndim == 4, "Wrong number of dimensions"
assert precip_netcdf.shape[0] == n_ens_members, "Wrong ensemble size"
assert precip_netcdf.shape[1] == n_timesteps, "Wrong number of lead times"
assert precip_netcdf.shape[2:] == field_shape, "Wrong field shape"
# assert that the saved output is the same as the original output
assert np.allclose(
precip_netcdf, precip_output, equal_nan=True
), "Wrong output values"
# assert that leadtimes and timestamps are as expected
td = timedelta(minutes=timestep)
leadtimes = [(i + 1) * timestep for i in range(n_timesteps)]
timestamps = [startdate + (i + 1) * td for i in range(n_timesteps)]
assert (metadata_netcdf["leadtimes"] == leadtimes).all(), "Wrong leadtimes"
assert (metadata_netcdf["timestamps"] == timestamps).all(), "Wrong timestamps"
def forecast(
precip,
precip_metadata,
velocity,
timesteps,
timestep,
nowcast_method,
precip_nwp=None,
precip_nwp_metadata=None,
start_blending=120,
end_blending=240,
fill_nwp=True,
nowcast_kwargs=None,
):
"""Generate a forecast by linearly blending nowcasts with NWP data
Parameters
----------
precip: array_like
Array containing the input precipitation field(s) ordered by timestamp
from oldest to newest. The time steps between the inputs are assumed
to be regular.
precip_metadata: dict
Metadata dictionary containing (at least) the transform, unit and threshold
attributes as described in the documentation of :py:mod:`pysteps.io.importers`.
velocity; array_like
Array of shape (2, m, n) containing the x- and y-components of the advection
field. The velocities are assumed to represent one time step between the
inputs. All values are required to be finite.
timesteps: int
Number of time steps to forecast.
timestep: int or float
The time difference (in minutes) between consecutive forecast fields.
nowcast_method: str
Name of the nowcasting method. See :py:mod:`pysteps.nowcasts.interface`
for the list of available methods.
precip_nwp: array_like or NoneType, optional
Array of shape (timesteps, m, n) in the case of no ensemble or
of shape (n_ens_members, timesteps, m, n) in the case of an ensemble
containing the NWP precipitation fields ordered by timestamp from oldest
to newest. The time steps between the inputs are assumed to be regular
(and identical to the time step between the nowcasts). If no NWP
data is given the value of precip_nwp is None and no blending will be performed.
precip_nwp_metadata: dict or NoneType, optional
NWP metadata dictionary containing (at least) the transform, unit and threshold
attributes as described in the documentation of :py:mod:`pysteps.io.importers`.
start_blending: int, optional
Time stamp (in minutes) after which the blending should start. Before this
only the nowcast data is used.
end_blending: int, optional
Time stamp (in minutes) after which the blending should end. Between
start_blending and end_blending the nowcasts and NWP data are linearly
merged with each other. After end_blending only the NWP data is used.
fill_nwp: bool, optional
Standard value is True. If True, the NWP data will be used to fill in the
no data mask of the nowcast.
nowcast_kwargs: dict, optional
Dictionary containing keyword arguments for the nowcast method.
Returns
-------
R_blended: ndarray
Array of shape (timesteps, m, n) in the case of no ensemble or
of shape (n_ens_members, timesteps, m, n) in the case of an ensemble
containing the precipation forecast generated by linearly blending
the nowcasts and the NWP data. n_ens_members equals the maximum no. of
ensemble members in either the nowcast or nwp model(s).
"""
if nowcast_kwargs is None:
nowcast_kwargs = dict()
# Calculate the nowcasts
nowcast_method_func = nowcasts.get_method(nowcast_method)
precip_nowcast = nowcast_method_func(
precip,
velocity,
timesteps,
**nowcast_kwargs,
)
# Make sure that precip_nowcast and precip_nwp are in mm/h
precip_nowcast, _ = conversion.to_rainrate(precip_nowcast, metadata=precip_metadata)
# Check if NWP data is given as input
if precip_nwp is not None:
precip_nwp, _ = conversion.to_rainrate(precip_nwp, metadata=precip_nwp_metadata)
if len(precip_nowcast.shape) == 4:
n_ens_members_nowcast = precip_nowcast.shape[0]
if n_ens_members_nowcast == 1:
precip_nowcast = np.squeeze(precip_nowcast)
else:
n_ens_members_nowcast = 1
if len(precip_nwp.shape) == 4:
n_ens_members_nwp = precip_nwp.shape[0]
if n_ens_members_nwp == 1:
precip_nwp = np.squeeze(precip_nwp)
else:
n_ens_members_nwp = 1
# Now, repeat the nowcast ensemble members or the nwp models/members until
# it has the same amount of members as n_ens_members_max. For instance, if
# you have 10 ensemble nowcasts members and 3 NWP members, the output will
# be an ensemble of 10 members. Hence, the three NWP members are blended
# with the first three members of the nowcast (member one with member one,
# two with two, etc.), subsequently, the same NWP members are blended with
# the next three members (NWP member one with member 4, NWP member 2 with
# member 5, etc.), until 10 is reached.
n_ens_members_max = max(n_ens_members_nowcast, n_ens_members_nwp)
n_ens_members_min = min(n_ens_members_nowcast, n_ens_members_nwp)
if n_ens_members_min != n_ens_members_max:
if n_ens_members_nwp == 1:
precip_nwp = np.repeat(
precip_nwp[np.newaxis, :, :], n_ens_members_max, axis=0
)
elif n_ens_members_nowcast == 1:
precip_nowcast = np.repeat(
precip_nowcast[np.newaxis, :, :], n_ens_members_max, axis=0
)
else:
repeats = [
(n_ens_members_max + i) // n_ens_members_min
for i in range(n_ens_members_min)
]
if n_ens_members_nwp == n_ens_members_min:
precip_nwp = np.repeat(precip_nwp, repeats, axis=0)
elif n_ens_members_nowcast == n_ens_members_min:
precip_nowcast = np.repeat(precip_nowcast, repeats, axis=0)
# Check if dimensions are correct
assert (
precip_nwp.shape == precip_nowcast.shape
), "The dimensions of precip_nowcast and precip_nwp need to be identical: dimension of precip_nwp = {} and dimension of precip_nowcast = {}".format(
precip_nwp.shape, precip_nowcast.shape
)
# Initialise output
R_blended = np.zeros_like(precip_nowcast)
# Calculate the weights
for i in range(timesteps):
# Calculate what time we are at
t = (i + 1) * timestep
# Calculate the weight with a linear relation (weight_nwp at start_blending = 0.0)
# and (weight_nwp at end_blending = 1.0)
weight_nwp = (t - start_blending) / (end_blending - start_blending)
# Set weights at times before start_blending and after end_blending
if weight_nwp < 0.0:
weight_nwp = 0.0
elif weight_nwp > 1.0:
weight_nwp = 1.0
# Calculate weight_nowcast
weight_nowcast = 1.0 - weight_nwp
# Calculate output by combining precip_nwp and precip_nowcast,
# while distinguishing between ensemble and non-ensemble methods
if n_ens_members_max == 1:
R_blended[i, :, :] = (
weight_nwp * precip_nwp[i, :, :]
+ weight_nowcast * precip_nowcast[i, :, :]
)
else:
R_blended[:, i, :, :] = (
weight_nwp * precip_nwp[:, i, :, :]
+ weight_nowcast * precip_nowcast[:, i, :, :]
)
# Find where the NaN values are and replace them with NWP data
if fill_nwp:
nan_indices = np.isnan(R_blended)
R_blended[nan_indices] = precip_nwp[nan_indices]
else:
# If no NWP data is given, the blended field is simply equal to the nowcast field
R_blended = precip_nowcast
return R_blended
if not np.any(np.isfinite(R[i, :, :])):
print("Skipping, no finite values found for time step %d" % (i+1))
missing_data = True
break
if missing_data:
curdate += timedelta(minutes=datasource["timestep"])
continue
R[~np.isfinite(R)] = metadata["zerovalue"]
R = transformation.dB_transform(R, metadata=metadata)[0]
oflow = motion.get_method("lucaskanade")
V = oflow(R[-2:, :, :])
nc = nowcasts.get_method("steps")
for es in ensemble_sizes:
for nw in range(1, max_num_threads+1, 1):
starttime = time.time()
_, init_time, mainloop_time = \
nc(R[-3:, :, :], V, num_timesteps, n_ens_members=es,
n_cascade_levels=6, R_thr=R_min_dB, kmperpixel=1.0,
timestep=datasource["timestep"], vel_pert_method="bps",
mask_method="incremental", probmatching_method="cdf",
num_workers=nw, fft_method=fft_method, measure_time=True)
results[es][nw] = (init_time, mainloop_time)
# Save the intermediate results for testing purposes.
with open("parallel_scaling_results_%s.dat" % domain, "wb") as f:
pickle.dump(dict(results), f)
def compute(nowcast_method, config_number):
# the optical flow methods to use
oflow_methods = ["darts"]
# time step between computation of each nowcast (minutes)
timestep = 30
# the number of time steps for each nowcast (5 minutes for the MeteoSwiss and
# FMI data)
num_timesteps = 24
# the threshold to use for precipitation/no precipitation
R_min = 0.1
R_min_dB = transformation.dB_transform(np.array([R_min]))[0][0]
precip_events = [
("201104160800", "201104170000"),
("201111152300", "201111161000"),
("201304110000", "201304120000"),
("201304041800", "201304051800"),
("201305180600", "201305181200"),
("201305270000", "201305271200"),
]
for pei, pe in enumerate(precip_events):
start_date = datetime.strptime(pe[0], "%Y%m%d%H%M")
curdate = datetime.strptime(pe[0], "%Y%m%d%H%M")
enddate = datetime.strptime(pe[1], "%Y%m%d%H%M")
results = {}
for m in oflow_methods:
results[m] = {}
results[m]["comptimes"] = 0.0
results[m]["CSI"] = [0.0] * num_timesteps
results[m]["RMSE"] = [0.0] * num_timesteps
results[m]["n_samples"] = [0.0] * num_timesteps
my_observations = LowAltCompositeCollection()
while curdate <= enddate:
print("Computing nowcasts for event %d, start date %s..." %
(pei + 1, str(curdate)),
end="")
sys.stdout.flush()
if curdate + num_timesteps * timedelta(minutes=5) > enddate:
break
time_step_in_sec = 5 * 60
times = [
curdate - timedelta(seconds=time_step_in_sec * i)
for i in range(9)
]
times += [
curdate + timedelta(seconds=time_step_in_sec * i)
for i in range(1, num_timesteps + 1)
]
times.sort()
# Add elements to the collection if they don't exists
for _time in times:
my_observations.add(get_lowalt_file(_time))
# First 9 times
R = my_observations.get_data('Reflectivity', date=times[:9])
R = dbz_to_r(R, a=300., b=1.5)
_R = list()
# The original data is at 1km resolutions
# Downscale to 5 km resolution by 5x5 averaging
for i in range(9):
_R.append(downscale_local_mean(R[i, :-1, :], (5, 5)))
R = np.asarray(_R)
my_observations.clean_buffers() # release memory
missing_data = False
for i in range(R.shape[0]):
if not np.any(np.isfinite(R[i, :, :])):
print("Skipping, no finite values found for time step %d" %
(i + 1))
missing_data = True
break
if missing_data:
curdate += timedelta(minutes=timestep)
continue
R = transformation.dB_transform(R)[0]
# Forecast times
fcts_times = times[9:]
R_obs = my_observations.get_data('Reflectivity', date=times[9:])
R_obs = dbz_to_r(R_obs, a=300., b=1.5)
# The original data is at 1km resolutions
# Downscale to 5 km resolution by 5x5 averaging
_R = list()
for i in range(len(fcts_times)):
_R.append(downscale_local_mean(R_obs[i, :-1, :], (5, 5)))
R_obs = np.asarray(_R)
my_observations.clean_buffers() # release memory
for oflow_method in oflow_methods:
oflow = motion.get_method(oflow_method)
if oflow_method == "vet":
R_ = R[-2:, :, :]
else:
R_ = R
starttime = time.time()
V = oflow(R_, **configurations[config_number])
print("%s optical flow computed in %.3f seconds." % \
(oflow_method, time.time() - starttime))
if nowcast_method == "advection":
nc = nowcasts.get_method("extrapolation")
R_fct = nc(R[-1, :, :], V, num_timesteps)
else:
nc = nowcasts.get_method("steps")
R_fct = nc(R[-3:, :, :],
V,
num_timesteps,
noise_method=None,
vel_pert_method=None,
n_ens_members=1,
mask_method="sprog",
R_thr=R_min_dB,
probmatching_method="mean",
fft_method="numpy")[0, :, :, :]
R_fct = transformation.dB_transform(R_fct, inverse=True)[0]
for lt in range(num_timesteps):
if not np.any(np.isfinite(R_obs[lt, :, :])):
print(
"Warning: no finite verifying observations for lead time %d."
% (lt + 1))
continue
csi = det_cat_fcst(R_fct[lt, :, :], R_obs[lt, :, :], R_min,
["CSI"])[0]
MASK = np.logical_and(R_fct[lt, :, :] > R_min,
R_obs[lt, :, :] > R_min)
if np.sum(MASK) == 0:
print("Skipping, no precipitation for lead time %d." %
(lt + 1))
continue
rmse = det_cont_fcst(R_fct[lt, :, :][MASK],
R_obs[lt, :, :][MASK], ["MAE_add"])[0]
results[oflow_method]["CSI"][lt] += csi
results[oflow_method]["RMSE"][lt] += rmse
results[oflow_method]["n_samples"][lt] += 1
print("Done.")
curdate += timedelta(minutes=timestep)
data_dir = './data/dart_tests/config_{:d}'.format(config_number)
create_dir(data_dir)
file_name = "optflow_comparison_results_%s_%s.dat" % (
nowcast_method, get_timestamp(start_date))
with open(join(data_dir, file_name), "wb") as f:
pickle.dump(results, f)
continue
for oflow_method in oflow_methods:
oflow = motion.get_method(oflow_method)
if oflow_method == "vet":
R_ = R[-2:, :, :]
else:
R_ = R
starttime = time.time()
V = oflow(R_)
print("%s optical flow computed in %.3f seconds." % \
(oflow_method, time.time() - starttime))
if nowcast_method == "advection":
nc = nowcasts.get_method("extrapolation")
R_fct = nc(R[-1, :, :], V, num_timesteps)
else:
nc = nowcasts.get_method("steps")
R_fct = nc(R[-3:, :, :],
V,
num_timesteps,
noise_method=None,
vel_pert_method=None,
n_ens_members=1,
mask_method="sprog",
R_thr=R_min_dB,
probmatching_method="mean",
fft_method="numpy")[0, :, :, :]
R_fct = transformation.dB_transform(R_fct, inverse=True)[0]
