def advection_correction(R, T=5, t=1):
"""
R = np.array([qpe_previous, qpe_current])
T = time between two observations (5 min)
t = interpolation timestep (1 min)
"""
# Evaluate advection
oflow_method = motion.get_method("LK")
fd_kwargs = {"buffer_mask": 10} # avoid edge effects
V = oflow_method(np.log(R), fd_kwargs=fd_kwargs)
# Perform temporal interpolation
Rd = np.zeros((R[0].shape))
x, y = np.meshgrid(
np.arange(R[0].shape[1], dtype=float),
np.arange(R[0].shape[0], dtype=float),
)
for i in range(1, 1 + int(T / t)):
pos1 = (y - i / T * V[1], x - i / T * V[0])
R1 = map_coordinates(R[0], pos1, order=1)
pos2 = (y + (T - i) / T * V[1], x + (T - i) / T * V[0])
R2 = map_coordinates(R[1], pos2, order=1)
Rd += (T - i) * R1 + i * R2
return 1 / T**2 * Rd
def test_steps_skill(
n_ens_members,
n_cascade_levels,
ar_order,
mask_method,
probmatching_method,
domain,
timesteps,
max_crps,
):
"""Tests STEPS nowcast skill."""
# inputs
precip_input, metadata = get_precipitation_fields(
num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(
num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000
)[1:, :, :]
precip_obs = precip_obs.filled()
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
nowcast_method = nowcasts.get_method("steps")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
timesteps=timesteps,
R_thr=metadata["threshold"],
kmperpixel=2.0,
timestep=metadata["accutime"],
seed=42,
n_ens_members=n_ens_members,
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
mask_method=mask_method,
probmatching_method=probmatching_method,
domain=domain,
)
assert precip_forecast.ndim == 4
assert precip_forecast.shape[0] == n_ens_members
assert precip_forecast.shape[1] == (
timesteps if isinstance(timesteps, int) else len(timesteps)
)
crps = verification.probscores.CRPS(precip_forecast[:, -1], precip_obs[-1])
assert crps < max_crps, f"CRPS={crps:.2f}, required < {max_crps:.2f}"
def test_steps(
n_ens_members,
n_cascade_levels,
ar_order,
mask_method,
probmatching_method,
domain,
max_crps,
):
"""Tests STEPS nowcast."""
# inputs
precip_input, metadata = get_precipitation_fields(
num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(num_prev_files=0,
num_next_files=3,
return_raw=False,
upscale=2000)[1:, :, :]
precip_obs = precip_obs.filled()
# Retrieve motion field
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
# Run nowcast
nowcast_method = nowcasts.get_method("steps")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
n_timesteps=3,
R_thr=metadata["threshold"],
kmperpixel=2.0,
timestep=metadata["accutime"],
seed=42,
n_ens_members=n_ens_members,
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
mask_method=mask_method,
probmatching_method=probmatching_method,
domain=domain,
)
# result
crps = verification.probscores.CRPS(precip_forecast[-1], precip_obs[-1])
print(f"got CRPS={crps:.1f}, required < {max_crps:.1f}")
assert crps < max_crps
def test_sprog(
n_cascade_levels,
ar_order,
probmatching_method,
domain,
min_csi):
"""Tests SPROG nowcast."""
# inputs
precip_input, metadata = get_precipitation_fields(num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(num_prev_files=0,
num_next_files=3,
return_raw=False,
upscale=2000)[1:, :, :]
precip_obs = precip_obs.filled()
# Retrieve motion field
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
# Run nowcast
nowcast_method = nowcasts.get_method("sprog")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
n_timesteps=3,
R_thr=metadata["threshold"],
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
probmatching_method=probmatching_method,
domain=domain,
)
# result
result = verification.det_cat_fct(
precip_forecast[-1],
precip_obs[-1],
thr=0.1,
scores="CSI")["CSI"]
print(f"got CSI={result:.1f}, required > {min_csi:.1f}")
assert result > min_csi
def DARTS(self,
history,
observation,
th_dbz,
n_leadtimes,
n_frames=3,
block_sizes=4,
win_sizes=8,
decl_scales=2):
oflow_method = motion.get_method("DARTS")
V = oflow_method(history)
extrapolate = nowcasts.get_method("extrapolation")
prediction_extrapolate = extrapolate(history[-1], V, n_leadtimes)
return prediction_extrapolate
def test_steps(
n_ens_members,
n_cascade_levels,
ar_order,
mask_method,
probmatching_method,
max_crps):
"""Tests STEPS nowcast."""
# inputs
precip_input, metadata = get_precipitation_fields(num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000)
precip_obs = get_precipitation_fields(num_prev_files=0,
num_next_files=3,
return_raw=False,
upscale=2000)[1:, :, :]
# Retrieve motion field
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
# Run nowcast
nowcast_method = nowcasts.get_method("steps")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
n_timesteps=3,
R_thr=metadata["threshold"],
kmperpixel=2.0,
timestep=metadata["accutime"],
seed=42,
n_ens_members=n_ens_members,
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
mask_method=mask_method,
probmatching_method=probmatching_method,
)
# result
result = verification.probscores.CRPS(precip_forecast[-1], precip_obs[-1])
assert result < max_crps
def test_sprog(
n_cascade_levels, ar_order, probmatching_method, domain, timesteps, min_csi
):
"""Tests SPROG nowcast."""
# inputs
precip_input, metadata = get_precipitation_fields(
num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(
num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000
)[1:, :, :]
precip_obs = precip_obs.filled()
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
nowcast_method = nowcasts.get_method("sprog")
precip_forecast = nowcast_method(
precip_input,
retrieved_motion,
timesteps=timesteps,
R_thr=metadata["threshold"],
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
probmatching_method=probmatching_method,
domain=domain,
)
assert precip_forecast.ndim == 3
assert precip_forecast.shape[0] == (
timesteps if isinstance(timesteps, int) else len(timesteps)
)
result = verification.det_cat_fct(
precip_forecast[-1], precip_obs[-1], thr=0.1, scores="CSI"
)["CSI"]
assert result > min_csi, f"CSI={result:.1f}, required > {min_csi:.1f}"
def test_anvil_rainrate(n_cascade_levels, ar_order, ar_window_radius,
timesteps, min_csi):
"""Tests ANVIL nowcast using rain rate precipitation fields."""
# inputs
precip_input, metadata = get_precipitation_fields(
num_prev_files=4,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip_input = precip_input.filled()
precip_obs = get_precipitation_fields(num_prev_files=0,
num_next_files=3,
return_raw=False,
upscale=2000)[1:, :, :]
precip_obs = precip_obs.filled()
pytest.importorskip("cv2")
oflow_method = motion.get_method("LK")
retrieved_motion = oflow_method(precip_input)
nowcast_method = nowcasts.get_method("anvil")
precip_forecast = nowcast_method(
precip_input[-(ar_order + 2):],
retrieved_motion,
timesteps=timesteps,
rainrate=None, # no R(VIL) conversion is done
n_cascade_levels=n_cascade_levels,
ar_order=ar_order,
ar_window_radius=ar_window_radius,
)
assert precip_forecast.ndim == 3
assert precip_forecast.shape[0] == (timesteps if isinstance(
timesteps, int) else len(timesteps))
result = verification.det_cat_fct(precip_forecast[-1],
precip_obs[-1],
thr=0.1,
scores="CSI")["CSI"]
assert result > min_csi, f"CSI={result:.2f}, required > {min_csi:.2f}"
def t_interp(t_boundary, img_boundary, analysis_time):
"""interpolate the two images temporally"""
tstart2t0_weight, t02tend_weight = \
get_t_weights(analysis_time, t_boundary)
dense_lucaskanade = motion.get_method('LK')
uv_forward = dense_lucaskanade(img_boundary)
uv_backward = dense_lucaskanade(img_boundary[::-1, :, :])
uv_forward, uv_backward = get_mean_uv(uv_forward, uv_backward)
extrapolate = nowcasts.get_method('extrapolation')
r_from_tstart = extrapolate(img_boundary[0, :, :],
uv_forward * t02tend_weight, 1)[0, :, :]
r_from_tend = extrapolate(img_boundary[1, :, :],
uv_backward * tstart2t0_weight, 1)[0, :, :]
r = r_from_tstart*tstart2t0_weight + \
r_from_tend*t02tend_weight
return r
def getV(self, history, n_frame, block_size, win_size, decl_scale):
oflow_method = motion.get_method("lucaskanade")
fd_kwargs = {}
fd_kwargs["max_corners"] = 1000
fd_kwargs["quality_level"] = 0.01
fd_kwargs["min_distance"] = 2
fd_kwargs["block_size"] = block_size # 4-8
lk_kwargs = {}
lk_kwargs["winsize"] = (win_size, win_size) # 5-10
oflow_kwargs = {}
oflow_kwargs["fd_kwargs"] = fd_kwargs
oflow_kwargs["lk_kwargs"] = lk_kwargs
oflow_kwargs["decl_scale"] = decl_scale # 2-4
V = oflow_method(history[-n_frame:], **oflow_kwargs) # 3 to 5 frames
return V
def test_visualization_motionfields_quiver(
source,
axis,
step,
quiver_kwargs,
map_kwargs,
upscale,
pass_geodata,
):
if source is not None:
fields, geodata = get_precipitation_fields(0, 2, False, True, upscale,
source)
if not pass_geodata:
geodata = None
ax = plot_precip_field(fields[-1], geodata=geodata)
oflow_method = motion.get_method("LK")
UV = oflow_method(fields)
else:
shape = (100, 100)
geodata = None
ax = None
u = np.ones(shape[1]) * shape[0]
v = np.arange(0, shape[0])
U, V = np.meshgrid(u, v)
UV = np.concatenate([U[None, :], V[None, :]])
UV_orig = UV.copy()
__ = quiver(
UV,
ax,
geodata,
axis,
step,
quiver_kwargs,
map_kwargs=map_kwargs,
)
# Check that quiver does not modify the input data
assert np.array_equal(UV, UV_orig)
def test_no_precipitation(optflow_method_name, num_times):
"""
Test that the motion methods work well with a zero precipitation in the
domain.
The expected result is a zero motion vector.
Parameters
----------
optflow_method_name: str
Optical flow method name
num_times : int
Number of precipitation frames (times) used as input for the optical
flow methods.
"""
zero_precip = np.zeros((num_times,) + reference_field.shape)
motion_method = motion.get_method(optflow_method_name)
uv_motion = motion_method(zero_precip, verbose=False)
assert np.abs(uv_motion).max() < 0.01
def test_visualization_motionfields_streamplot(
source, axis, streamplot_kwargs, map_kwargs, upscale, pass_geodata
):
if source is not None:
fields, geodata = get_precipitation_fields(0, 2, False, True, upscale, source)
if not pass_geodata:
pass_geodata = None
ax = plot_precip_field(fields[-1], geodata=geodata)
oflow_method = motion.get_method("LK")
UV = oflow_method(fields)
else:
shape = (100, 100)
geodata = None
ax = None
u = np.ones(shape[1]) * shape[0]
v = np.arange(0, shape[0])
U, V = np.meshgrid(u, v)
UV = np.concatenate([U[None, :], V[None, :]])
__ = streamplot(UV, ax, geodata, axis, streamplot_kwargs, map_kwargs=map_kwargs,)
def test_visualization_motionfields_quiver(
source,
map,
drawlonlatlines,
lw,
axis,
step,
quiver_kwargs,
upscale,
):
if map == "cartopy":
pytest.importorskip("cartopy")
elif map == "basemap":
pytest.importorskip("basemap")
if source is not None:
fields, geodata = get_precipitation_fields(0, 2, False, True, upscale,
source)
ax = plot_precip_field(
fields[-1],
map=map,
geodata=geodata,
)
oflow_method = motion.get_method("LK")
UV = oflow_method(fields)
else:
shape = (100, 100)
geodata = None
ax = None
u = np.ones(shape[1]) * shape[0]
v = np.arange(0, shape[0])
U, V = np.meshgrid(u, v)
UV = np.concatenate([U[None, :], V[None, :]])
__ = quiver(UV, ax, map, geodata, drawlonlatlines, lw, axis, step,
quiver_kwargs)
def test_lk_masked_array():
"""
Passing a ndarray with NaNs or a masked array should produce the same results.
"""
pytest.importorskip('cv2')
__, precip_obs = _create_observations(reference_field.copy(),
'linear_y',
num_times=2)
motion_method = motion.get_method('LK')
# ndarray with nans
np.ma.set_fill_value(precip_obs, -15)
ndarray = precip_obs.filled()
ndarray[ndarray == -15] = np.nan
uv_ndarray = motion_method(ndarray, fd_kwargs={'buffer_mask':20}, verbose=False)
# masked array
mdarray = np.ma.masked_invalid(ndarray)
mdarray.data[mdarray.mask] = -15
uv_mdarray = motion_method(mdarray, fd_kwargs={'buffer_mask':20}, verbose=False)
assert np.abs(uv_mdarray - uv_ndarray).max() < 0.01
def test_input_shape_checks(optflow_method_name, minimum_input_frames,
maximum_input_frames):
image_size = 100
motion_method = motion.get_method(optflow_method_name)
if maximum_input_frames == np.inf:
maximum_input_frames = minimum_input_frames + 10
with not_raises(Exception):
for frames in range(minimum_input_frames, maximum_input_frames + 1):
motion_method(np.zeros((frames, image_size, image_size)),
verbose=False)
with pytest.raises(ValueError):
motion_method(np.zeros((2, )))
motion_method(np.zeros((2, 2)))
for frames in range(minimum_input_frames):
motion_method(np.zeros((frames, image_size, image_size)),
verbose=False)
for frames in range(maximum_input_frames + 1,
maximum_input_frames + 4):
motion_method(np.zeros((frames, image_size, image_size)),
verbose=False)
def test_optflow_method_convergence(input_precip, optflow_method_name,
motion_type, num_times, max_rel_rmse):
"""
Test the convergence to the actual solution of the optical flow method used.
We measure the error in the retrieved field by using the
Relative RMSE = Rel_RMSE = sqrt(Relative MSE)
- Rel_RMSE = 0%: no error
- Rel_RMSE = 100%: The retrieved motion field has an average error
equal in magnitude to the motion field.
Relative RMSE is computed only un a region surrounding the precipitation
field, were we have enough information to retrieve the motion field.
The precipitation region includes the precipitation pattern plus a margin
of approximately 20 grid points.
Parameters
----------
input_precip: numpy array (lat, lon)
Input precipitation field.
optflow_method_name: str
Optical flow method name
motion_type : str
The supported motion fields are:
- linear_x: (u=2, v=0)
- linear_y: (u=0, v=2)
"""
ideal_motion, precip_obs = _create_observations(input_precip.copy(),
motion_type,
num_times=num_times)
oflow_method = motion.get_method(optflow_method_name)
if optflow_method_name == 'vet':
# By default, the maximum number of iteration in the VET minimization
# is maxiter=100.
# To increase the stability of the tests to we increase this value to
# maxiter=150.
computed_motion = oflow_method(precip_obs, verbose=False,
options=dict(maxiter=150))
else:
computed_motion = oflow_method(precip_obs, verbose=False)
precip_data, _ = stp.utils.dB_transform(precip_obs.max(axis=0),
inverse=True)
precip_data.data[precip_data.mask] = 0
precip_mask = ((uniform_filter(precip_data, size=20) > 0.1)
& ~precip_obs.mask.any(axis=0))
# To evaluate the accuracy of the computed_motion vectors, we will use
# a relative RMSE measure.
# Relative MSE = < (expected_motion - computed_motion)^2 > / <expected_motion^2 >
# Relative RMSE = sqrt(Relative MSE)
mse = ((ideal_motion - computed_motion)[:, precip_mask] ** 2).mean()
rel_mse = mse / (ideal_motion[:, precip_mask] ** 2).mean()
rel_rmse = np.sqrt(rel_mse) * 100
print(f"method:{optflow_method_name} ; "
f"motion:{motion_type} ; times: {num_times} ; "
f"rel_rmse:{rel_rmse:.2f}%")
assert rel_rmse < max_rel_rmse
#
# Also note that the radar image includes NaNs in areas of missing data.
# These are used by the optical flow algorithm to define the radar mask.
#
# Sparse Lucas-Kanade
# -------------------
#
# By setting the optional argument ``dense=False`` in ``xy, uv = dense_lucaskanade(...)``,
# the LK algorithm returns the motion vectors detected by the Lucas-Kanade scheme
# without interpolating them on the grid.
# This allows us to better identify the presence of wrongly detected
# stationary motion in areas where precipitation is leaving the domain (look
# for the red dots within the blue circle in the figure below).
# Get Lucas-Kanade optical flow method
dense_lucaskanade = motion.get_method("LK")
# Mask invalid values
R = np.ma.masked_invalid(R)
# Use no buffering of the radar mask
fd_kwargs1 = {"buffer_mask": 0}
xy, uv = dense_lucaskanade(R, dense=False, fd_kwargs=fd_kwargs1)
plt.imshow(ref_dbr, cmap=plt.get_cmap("Greys"))
plt.imshow(mask, cmap=colors.ListedColormap(["black"]), alpha=0.5)
plt.quiver(
xy[:, 0],
xy[:, 1],
uv[:, 0],
uv[:, 1],
color="red",
# Listing 2
from matplotlib import pyplot
from pysteps import motion, nowcasts
from pysteps.postprocessing.ensemblestats import excprob
from pysteps.utils import transformation
from pysteps.visualization import plot_precip_field
# compute the advection field
oflow_method = motion.get_method("lucaskanade")
V = oflow_method(R)
# compute the S-PROG nowcast
nowcast_method = nowcasts.get_method("sprog")
R_f_sprog = nowcast_method(R[-3:, :, :], V, 12, R_thr=-10.0)[-1, :, :]
# compute the STEPS nowcast
nowcast_method = nowcasts.get_method("steps")
R_f = nowcast_method(R[-3:, :, :],
V,
12,
n_ens_members=24,
n_cascade_levels=8,
R_thr=-10.0,
kmperpixel=1.0,
timestep=5)
# plot the S-PROG nowcast, one ensemble member of the STEPS nowcast and the exceedance
# probability of 0.1 mm/h computed from the ensemble
R_f_sprog = transformation.dB_transform(R_f_sprog,
threshold=-10.0,
def compute(nowcast_method, config_number):
# the optical flow methods to use
oflow_methods = ["darts"]
# time step between computation of each nowcast (minutes)
timestep = 30
# the number of time steps for each nowcast (5 minutes for the MeteoSwiss and
# FMI data)
num_timesteps = 24
# the threshold to use for precipitation/no precipitation
R_min = 0.1
R_min_dB = transformation.dB_transform(np.array([R_min]))[0][0]
precip_events = [
("201104160800", "201104170000"),
("201111152300", "201111161000"),
("201304110000", "201304120000"),
("201304041800", "201304051800"),
("201305180600", "201305181200"),
("201305270000", "201305271200"),
]
for pei, pe in enumerate(precip_events):
start_date = datetime.strptime(pe[0], "%Y%m%d%H%M")
curdate = datetime.strptime(pe[0], "%Y%m%d%H%M")
enddate = datetime.strptime(pe[1], "%Y%m%d%H%M")
results = {}
for m in oflow_methods:
results[m] = {}
results[m]["comptimes"] = 0.0
results[m]["CSI"] = [0.0] * num_timesteps
results[m]["RMSE"] = [0.0] * num_timesteps
results[m]["n_samples"] = [0.0] * num_timesteps
my_observations = LowAltCompositeCollection()
while curdate <= enddate:
print("Computing nowcasts for event %d, start date %s..." %
(pei + 1, str(curdate)),
end="")
sys.stdout.flush()
if curdate + num_timesteps * timedelta(minutes=5) > enddate:
break
time_step_in_sec = 5 * 60
times = [
curdate - timedelta(seconds=time_step_in_sec * i)
for i in range(9)
]
times += [
curdate + timedelta(seconds=time_step_in_sec * i)
for i in range(1, num_timesteps + 1)
]
times.sort()
# Add elements to the collection if they don't exists
for _time in times:
my_observations.add(get_lowalt_file(_time))
# First 9 times
R = my_observations.get_data('Reflectivity', date=times[:9])
R = dbz_to_r(R, a=300., b=1.5)
_R = list()
# The original data is at 1km resolutions
# Downscale to 5 km resolution by 5x5 averaging
for i in range(9):
_R.append(downscale_local_mean(R[i, :-1, :], (5, 5)))
R = np.asarray(_R)
my_observations.clean_buffers() # release memory
missing_data = False
for i in range(R.shape[0]):
if not np.any(np.isfinite(R[i, :, :])):
print("Skipping, no finite values found for time step %d" %
(i + 1))
missing_data = True
break
if missing_data:
curdate += timedelta(minutes=timestep)
continue
R = transformation.dB_transform(R)[0]
# Forecast times
fcts_times = times[9:]
R_obs = my_observations.get_data('Reflectivity', date=times[9:])
R_obs = dbz_to_r(R_obs, a=300., b=1.5)
# The original data is at 1km resolutions
# Downscale to 5 km resolution by 5x5 averaging
_R = list()
for i in range(len(fcts_times)):
_R.append(downscale_local_mean(R_obs[i, :-1, :], (5, 5)))
R_obs = np.asarray(_R)
my_observations.clean_buffers() # release memory
for oflow_method in oflow_methods:
oflow = motion.get_method(oflow_method)
if oflow_method == "vet":
R_ = R[-2:, :, :]
else:
R_ = R
starttime = time.time()
V = oflow(R_, **configurations[config_number])
print("%s optical flow computed in %.3f seconds." % \
(oflow_method, time.time() - starttime))
if nowcast_method == "advection":
nc = nowcasts.get_method("extrapolation")
R_fct = nc(R[-1, :, :], V, num_timesteps)
else:
nc = nowcasts.get_method("steps")
R_fct = nc(R[-3:, :, :],
V,
num_timesteps,
noise_method=None,
vel_pert_method=None,
n_ens_members=1,
mask_method="sprog",
R_thr=R_min_dB,
probmatching_method="mean",
fft_method="numpy")[0, :, :, :]
R_fct = transformation.dB_transform(R_fct, inverse=True)[0]
for lt in range(num_timesteps):
if not np.any(np.isfinite(R_obs[lt, :, :])):
print(
"Warning: no finite verifying observations for lead time %d."
% (lt + 1))
continue
csi = det_cat_fcst(R_fct[lt, :, :], R_obs[lt, :, :], R_min,
["CSI"])[0]
MASK = np.logical_and(R_fct[lt, :, :] > R_min,
R_obs[lt, :, :] > R_min)
if np.sum(MASK) == 0:
print("Skipping, no precipitation for lead time %d." %
(lt + 1))
continue
rmse = det_cont_fcst(R_fct[lt, :, :][MASK],
R_obs[lt, :, :][MASK], ["MAE_add"])[0]
results[oflow_method]["CSI"][lt] += csi
results[oflow_method]["RMSE"][lt] += rmse
results[oflow_method]["n_samples"][lt] += 1
print("Done.")
curdate += timedelta(minutes=timestep)
data_dir = './data/dart_tests/config_{:d}'.format(config_number)
create_dir(data_dir)
file_name = "optflow_comparison_results_%s_%s.dat" % (
nowcast_method, get_timestamp(start_date))
with open(join(data_dir, file_name), "wb") as f:
pickle.dump(results, f)
def test_lk(
lk_kwargs,
fd_method,
dense,
nr_std_outlier,
k_outlier,
size_opening,
decl_scale,
verbose,
):
"""Tests Lucas-Kanade optical flow."""
pytest.importorskip("cv2")
if fd_method == "blob":
pytest.importorskip("skimage")
if fd_method == "tstorm":
pytest.importorskip("skimage")
pytest.importorskip("pandas")
# inputs
precip, metadata = get_precipitation_fields(
num_prev_files=2,
num_next_files=0,
return_raw=False,
metadata=True,
upscale=2000,
)
precip = precip.filled()
# Retrieve motion field
oflow_method = motion.get_method("LK")
output = oflow_method(
precip,
lk_kwargs=lk_kwargs,
fd_method=fd_method,
dense=dense,
nr_std_outlier=nr_std_outlier,
k_outlier=k_outlier,
size_opening=size_opening,
decl_scale=decl_scale,
verbose=verbose,
)
# Check format of ouput
if dense:
assert isinstance(output, np.ndarray)
assert output.ndim == 3
assert output.shape[0] == 2
assert output.shape[1:] == precip[0].shape
if nr_std_outlier == 0:
assert output.sum() == 0
else:
assert isinstance(output, tuple)
assert len(output) == 2
assert isinstance(output[0], np.ndarray)
assert isinstance(output[1], np.ndarray)
assert output[0].ndim == 2
assert output[1].ndim == 2
assert output[0].shape[1] == 2
assert output[1].shape[1] == 2
assert output[0].shape[0] == output[1].shape[0]
if nr_std_outlier == 0:
assert output[0].shape[0] == 0
assert output[1].shape[0] == 0
# Store the last frame for plotting it later later
R_ = R[-1, :, :].copy()
# Log-transform the data to unit of dBR, set the threshold to 0.1 mm/h,
# set the fill value to -15 dBR
R, metadata = transformation.dB_transform(R,
metadata,
threshold=0,
zerovalue=-15.0)
# Nicely print the metadata
pprint(metadata)
# Estimate the motion field with Lucas-Kanade
oflow_method = motion.get_method("LK")
V = oflow_method(R[-3:, :, :])
# Extrapolate the last radar observation
extrapolate = nowcasts.get_method("extrapolation")
R[~np.isfinite(R)] = metadata["zerovalue"]
R_f = extrapolate(R[-1, :, :], V, n_leadtimes)
# Back-transform to rain rate
R_f = transformation.dB_transform(R_f, threshold=0.1, inverse=True)[0]
# Plot the motion field
animations.animate(R,
nloops=1,
R_fct=R_f,
timestep_min=5,
#
# Also note that the radar image includes NaNs in areas of missing data.
# These are used by the optical flow algorithm to define the radar mask.
#
# Sparse Lucas-Kanade
# -------------------
#
# By setting the optional argument 'dense=False' in 'x,y,u,v = LK_optflow(.....)',
# the LK algorithm returns the motion vectors detected by the Lucas-Kanade scheme
# without interpolating them on the grid.
# This allows us to better identify the presence of wrongly detected
# stationary motion in areas where precipitation is leaving the domain (look
# for the red dots within the blue circle in the figure below).
# get Lucas-Kanade optical flow method
LK_optflow = motion.get_method("LK")
# Mask invalid values
R = np.ma.masked_invalid(R)
R.data[R.mask] = np.nan
# Use default settings (i.e., no buffering of the radar mask)
x, y, u, v = LK_optflow(R, dense=False, buffer_mask=0, quality_level_ST=0.1)
plt.imshow(ref_dbr, cmap=plt.get_cmap("Greys"))
plt.imshow(mask, cmap=colors.ListedColormap(["black"]), alpha=0.5)
plt.quiver(x, y, u, v, color="red", angles="xy", scale_units="xy", scale=0.2)
circle = plt.Circle((620, 245), 100, color="b", clip_on=False, fill=False)
plt.gca().add_artist(circle)
plt.title("buffer_mask = 0 (default)")
plt.show()
zerovalue=-15.0)
# Nicely print the metadata
pprint(metadata)
################################################################################
# Lucas-Kanade (LK)
# -----------------
#
# The Lucas-Kanade optical flow method implemented in pysteps is a local
# tracking approach that relies on the OpenCV package.
# Local features are tracked in a sequence of two or more radar images. The
# scheme includes a final interpolation step in order to produce a smooth
# field of motion vectors.
oflow_method = motion.get_method("LK")
V1 = oflow_method(R[-3:, :, :])
# Plot the motion field on top of the reference frame
plot_precip_field(R_, geodata=metadata, title="LK")
quiver(V1, geodata=metadata, step=25)
plt.show()
################################################################################
# Variational echo tracking (VET)
# -------------------------------
#
# This module implements the VET algorithm presented
# by Laroche and Zawadzki (1995) and used in the McGill Algorithm for
# Prediction by Lagrangian Extrapolation (MAPLE) described in
# Germann and Zawadzki (2002).
def dating(
input_video,
timelist,
mintrack=3,
cell_list=None,
label_list=None,
start=0,
minref=35,
maxref=48,
mindiff=6,
minsize=50,
minmax=41,
mindis=10,
dyn_thresh=False,
):
"""
This function performs the thunderstorm detection and tracking DATing.
It requires a 3-D input array that contains the temporal succession of the 2-D data
array of each timestep. On each timestep the detection is performed, the identified
objects are advected with a flow prediction and the advected objects are matched to
the newly identified objects of the next timestep.
The last portion re-arranges the data into tracks sorted by ID-number.
Parameters
----------
input_video : array-like
Array of shape (t,m,n) containing input image, with t being the temporal
dimension and m,n the spatial dimensions. Thresholds are tuned to maximum
reflectivity in dBZ with a spatial resolution of 1 km and a temporal resolution
of 5 min. Nan values are ignored.
timelist : list
List of length t containing string of time and date of each (m,n) field.
mintrack : int, optional
minimum duration of cell-track to be counted. The default is 3 time steps.
cell_list : list or None, optional
If you wish to expand an existing list of cells, insert previous cell-list here.
The default is None.
If not None, requires that label_list has the same length.
label_list : list or None, optional
If you wish to expand an existing list of cells, insert previous label-list here.
The default is None.
If not None, requires that cell_list has the same length.
start : int, optional
If you wish to expand an existing list of cells, the input video must contain 2
timesteps prior to the merging. The start can then be set to 2, allowing the
motion vectors to be formed from the first three grids and continuing the cell
tracking from there. The default is 0, which initiates a new tracking sequence.
minref : float, optional
Lower threshold for object detection. Lower values will be set to NaN.
The default is 35 dBZ.
maxref : float, optional
Upper threshold for object detection. Higher values will be set to this value.
The default is 48 dBZ.
mindiff : float, optional
Minimal difference between two identified maxima within same area to split area
into two objects. The default is 6 dBZ.
minsize : float, optional
Minimal area for possible detected object. The default is 50 pixels.
minmax : float, optional
Minimum value of maximum in identified objects. Objects with a maximum lower
than this will be discarded. The default is 41 dBZ.
mindis : float, optional
Minimum distance between two maxima of identified objects. Objects with a
smaller distance will be merged. The default is 10 km.
Returns
-------
track_list : list of dataframes
Each dataframe contains the track and properties belonging to one cell ID.
Columns of dataframes: ID - cell ID, time - time stamp, x - array of all
x-coordinates of cell, y - array of all y-coordinates of cell, cen_x -
x-coordinate of cell centroid, cen_y - y-coordinate of cell centroid, max_ref -
maximum (reflectivity) value of cell, cont - cell contours
cell_list : list of dataframes
Each dataframe contains the detected cells and properties belonging to one
timestep. The IDs are already matched to provide a track.
Columns of dataframes: ID - cell ID, time - time stamp, x - array of all
x-coordinates of cell, y - array of all y-coordinates of cell, cen_x -
x-coordinate of cell centroid, cen_y - y-coordinate of cell centroid, max_ref -
maximum (reflectivity) value of cell, cont - cell contours
label_list : list of arrays
Each (n,m) array contains the gridded IDs of the cells identified in the
corresponding timestep. The IDs are already matched to provide a track.
"""
if not SKIMAGE_IMPORTED:
raise MissingOptionalDependency(
"skimage is required for thunderstorm DATing "
"but it is not installed")
if not PANDAS_IMPORTED:
raise MissingOptionalDependency(
"pandas is required for thunderstorm DATing "
"but it is not installed")
# Check arguments
if cell_list is None or label_list is None:
cell_list = []
label_list = []
else:
if not len(cell_list) == len(label_list):
raise ValueError("len(cell_list) != len(label_list)")
if start > len(timelist):
raise ValueError("start > len(timelist)")
oflow_method = motion.get_method("LK")
max_ID = 0
for t in range(start, len(timelist)):
cells_id, labels = tstorm_detect.detection(
input_video[t, :, :],
minref=minref,
maxref=maxref,
mindiff=mindiff,
minsize=minsize,
minmax=minmax,
mindis=mindis,
time=timelist[t],
)
if len(cell_list) < 2:
cell_list.append(cells_id)
label_list.append(labels)
cid = np.unique(labels)
max_ID = np.nanmax([np.nanmax(cid), max_ID]) + 1
continue
if t >= 2:
flowfield = oflow_method(input_video[t - 2:t + 1, :, :])
cells_id, max_ID, newlabels = tracking(cells_id, cell_list[-1],
labels, flowfield, max_ID)
cid = np.unique(newlabels)
# max_ID = np.nanmax([np.nanmax(cid), max_ID])
cell_list.append(cells_id)
label_list.append(newlabels)
track_list = couple_track(cell_list[2:], int(max_ID), mintrack)
return track_list, cell_list, label_list
zerovalue=-15.0)
# Nicely print the metadata
pprint(metadata)
################################################################################
# Lucas-Kanade (LK)
# -----------------
#
# The Lucas-Kanade optical flow method implemented in pysteps is a local
# tracking approach that relies on the OpenCV package.
# Local features are tracked in a sequence of two or more radar images. The
# scheme includes a final interpolation step in order to produce a smooth
# field of motion vectors.
oflow_method = motion.get_method("LK")
V1 = oflow_method(R[-3:, :, :])
# Plot the motion field on top of the reference frame
plot_precip_field(R_, geodata=metadata, title="LK")
quiver(V1, geodata=metadata, step=25)
plt.show()
################################################################################
# Variational echo tracking (VET)
# -------------------------------
#
# This module implements the VET algorithm presented
# by Laroche and Zawadzki (1995) and used in the McGill Algorithm for
# Prediction by Lagrangian Extrapolation (MAPLE) described in
# Germann and Zawadzki (2002).
nargs='?',
type=str,
metavar="filename",
help="accumulate statistics to previously computed file <filename>")
args = argparser.parse_args()
datasource = rcparams["data_sources"][args.datasource]
startdate = datetime.strptime(args.startdate, "%Y%m%d%H%M")
enddate = datetime.strptime(args.enddate, "%Y%m%d%H%M")
importer = io.get_method(datasource["importer"], "importer")
motionfields = {}
oflow = motion.get_method(args.oflow)
# compute motion fields
# ---------------------
# TODO: This keeps all motion fields in memory during the analysis period, which
# can take a lot of memory.
curdate = startdate
while curdate <= enddate:
try:
fns = io.archive.find_by_date(curdate,
datasource["root_path"],
datasource["path_fmt"],
datasource["fn_pattern"],
datasource["fn_ext"],
def plot_optflow_method_convergence(input_precip,
optflow_method_name,
motion_type):
"""
Test the convergence to the actual solution of the optical flow method used.
Parameters
----------
input_precip: numpy array (lat, lon)
Input precipitation field.
optflow_method_name: str
Optical flow method name
motion_type : str
The supported motion fields are:
- linear_x: (u=2, v=0)
- linear_y: (u=0, v=2)
- rotor: rotor field
"""
if optflow_method_name.lower() != "darts":
num_times = 2
else:
num_times = 9
ideal_motion, precip_obs = create_observations(input_precip,
motion_type,
num_times=num_times)
oflow_method = motion.get_method(optflow_method_name)
computed_motion = oflow_method(precip_obs, verbose=False)
precip_obs, _ = stp.utils.dB_transform(precip_obs, inverse=True)
precip_data = precip_obs.max(axis=0)
precip_data.data[precip_data.mask] = 0
precip_mask = ((uniform_filter(precip_data, size=20) > 0.1)
& ~precip_obs.mask.any(axis=0))
cmap = get_cmap('jet')
cmap.set_under('grey', alpha=0.25)
cmap.set_over('none')
# Compare retrieved motion field with the ideal one
plt.figure(figsize=(9, 4))
plt.subplot(1, 2, 1)
ax = plot_precip_field(precip_obs[0], title="Reference motion")
quiver(ideal_motion, step=25, ax=ax)
plt.subplot(1, 2, 2)
ax = plot_precip_field(precip_obs[0], title="Retrieved motion")
quiver(computed_motion, step=25, ax=ax)
# To evaluate the accuracy of the computed_motion vectors, we will use
# a relative RMSE measure.
# Relative MSE = < (expected_motion - computed_motion)^2 > / <expected_motion^2 >
# Relative RMSE = sqrt(Relative MSE)
mse = ((ideal_motion - computed_motion)[:, precip_mask] ** 2).mean()
rel_mse = mse / (ideal_motion[:, precip_mask] ** 2).mean()
plt.suptitle(f"{optflow_method_name} "
f"Relative RMSE: {np.sqrt(rel_mse) * 100:.2f}%")
plt.show()
results = {}
results["leadtimes"] = (np.arange(num_timesteps) + 1) * datasource["timestep"]
results["timestep"] = datasource["timestep"]
results["cc_ar"] = [np.zeros(num_timesteps) for i in range(num_cascade_levels)]
results["cc_obs"] = [
np.zeros(num_timesteps) for i in range(num_cascade_levels)
]
results["n_ar_samples"] = [np.zeros(num_timesteps, dtype=int) \
for i in range(num_cascade_levels)]
results["n_obs_samples"] = [np.zeros(num_timesteps, dtype=int) \
for i in range(num_cascade_levels)]
filter = None
oflow = motion.get_method(oflow_method)
extrapolator = extrapolation.get_method("semilagrangian")
for pei, pe in enumerate(precipevents):
curdate = datetime.strptime(pe[0], "%Y%m%d%H%M")
enddate = datetime.strptime(pe[1], "%Y%m%d%H%M")
while curdate <= enddate:
print("Running analyses for event %d, start date %s..." %
(pei + 1, str(curdate)),
end="")
sys.stdout.flush()
if curdate + num_timesteps * timedelta(minutes=5) > enddate:
print("Done.")
break