def get_obs_fcst_TRT_Rank(TRT_t0, TRT_diff_pred, TRT_diff_obs, TRT_tneg5):
obs = TRT_t0 + TRT_diff_obs
pred_mod = TRT_t0 + TRT_diff_pred
pred_mod.name = "Rank model prediction"
pred_mod_PM = pd.Series(nonparam_match_empirical_cdf(
pred_mod.values, obs.values),
index=pred_mod.index,
name="Rank model prediction (PM)")
pred_pers = TRT_t0.copy()
pred_pers.name = "Rank persistency prediction"
pred_pers_PM = pd.Series(nonparam_match_empirical_cdf(
pred_pers.values, obs.values),
index=pred_pers.index,
name="Rank persistency prediction (PM)")
pred_diff = TRT_t0 + (TRT_t0 - TRT_tneg5)
pred_diff.name = "Rank constant gradient prediction"
diff_pred = pd.Series(TRT_diff_pred,
index=TRT_t0.index,
name="TRT rank difference model prediction")
return (obs, pred_mod, pred_mod_PM, pred_pers, pred_pers_PM, pred_diff,
diff_pred)
def forecast(
R,
V,
n_timesteps,
n_cascade_levels=6,
R_thr=None,
extrap_method="semilagrangian",
decomp_method="fft",
bandpass_filter_method="gaussian",
ar_order=2,
conditional=False,
probmatching_method="cdf",
num_workers=1,
fft_method="numpy",
domain="spatial",
extrap_kwargs=None,
filter_kwargs=None,
measure_time=False,
):
"""Generate a nowcast by using the Spectral Prognosis (S-PROG) method.
Parameters
----------
R : array-like
Array of shape (ar_order+1,m,n) containing the input precipitation fields
ordered by timestamp from oldest to newest. The time steps between
the inputs are assumed to be regular.
V : array-like
Array of shape (2,m,n) containing the x- and y-components of the
advection field.
The velocities are assumed to represent one time step between the
inputs. All values are required to be finite.
n_timesteps : int
Number of time steps to forecast.
n_cascade_levels : int, optional
The number of cascade levels to use.
R_thr : float
The threshold value for minimum observable precipitation intensity.
extrap_method : str, optional
Name of the extrapolation method to use. See the documentation of
pysteps.extrapolation.interface.
decomp_method : {'fft'}, optional
Name of the cascade decomposition method to use. See the documentation
of pysteps.cascade.interface.
bandpass_filter_method : {'gaussian', 'uniform'}, optional
Name of the bandpass filter method to use with the cascade decomposition.
See the documentation of pysteps.cascade.interface.
ar_order : int, optional
The order of the autoregressive model to use. Must be >= 1.
conditional : bool, optional
If set to True, compute the statistics of the precipitation field
conditionally by excluding pixels where the values are
below the threshold R_thr.
probmatching_method : {'cdf','mean',None}, optional
Method for matching the conditional statistics of the forecast field
(areas with precipitation intensity above the threshold R_thr) with those
of the most recently observed one. 'cdf'=map the forecast CDF to the
observed one, 'mean'=adjust only the mean value,
None=no matching applied.
num_workers : int, optional
The number of workers to use for parallel computation. Applicable if dask
is enabled or pyFFTW is used for computing the FFT.
When num_workers>1, it is advisable to disable OpenMP by setting
the environment variable OMP_NUM_THREADS to 1.
This avoids slowdown caused by too many simultaneous threads.
fft_method : str, optional
A string defining the FFT method to use (see utils.fft.get_method).
Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,
the recommended method is 'pyfftw'.
domain : {"spatial", "spectral"}
If "spatial", all computations are done in the spatial domain (the
classical S-PROG model). If "spectral", the AR(2) models are applied
directly in the spectral domain to reduce memory footprint and improve
performance :cite:`PCH2019a`.
extrap_kwargs : dict, optional
Optional dictionary containing keyword arguments for the extrapolation
method. See the documentation of pysteps.extrapolation.
filter_kwargs : dict, optional
Optional dictionary containing keyword arguments for the filter method.
See the documentation of pysteps.cascade.bandpass_filters.py.
measure_time : bool
If set to True, measure, print and return the computation time.
Returns
-------
out : ndarray
A three-dimensional array of shape (n_timesteps,m,n) containing a time
series of forecast precipitation fields. The time series starts from
t0+timestep, where timestep is taken from the input precipitation fields
R. If measure_time is True, the return value is a three-element tuple
containing the nowcast array, the initialization time of the nowcast
generator and the time used in the main loop (seconds).
See also
--------
pysteps.extrapolation.interface, pysteps.cascade.interface
References
----------
:cite:`Seed2003`, :cite:`PCH2019a`
"""
_check_inputs(R, V, ar_order)
if extrap_kwargs is None:
extrap_kwargs = dict()
if filter_kwargs is None:
filter_kwargs = dict()
if np.any(~np.isfinite(V)):
raise ValueError("V contains non-finite values")
print("Computing S-PROG nowcast:")
print("-------------------------")
print("")
print("Inputs:")
print("-------")
print("input dimensions: %dx%d" % (R.shape[1], R.shape[2]))
print("")
print("Methods:")
print("--------")
print("extrapolation: %s" % extrap_method)
print("bandpass filter: %s" % bandpass_filter_method)
print("decomposition: %s" % decomp_method)
print("conditional statistics: %s" % ("yes" if conditional else "no"))
print("probability matching: %s" % probmatching_method)
print("FFT method: %s" % fft_method)
print("domain: %s" % domain)
print("")
print("Parameters:")
print("-----------")
print("number of time steps: %d" % n_timesteps)
print("parallel threads: %d" % num_workers)
print("number of cascade levels: %d" % n_cascade_levels)
print("order of the AR(p) model: %d" % ar_order)
print("precip. intensity threshold: %g" % R_thr)
if measure_time:
starttime_init = time.time()
fft = utils.get_method(fft_method,
shape=R.shape[1:],
n_threads=num_workers)
M, N = R.shape[1:]
# initialize the band-pass filter
filter_method = cascade.get_method(bandpass_filter_method)
filter = filter_method((M, N), n_cascade_levels, **filter_kwargs)
decomp_method, recomp_method = cascade.get_method(decomp_method)
extrapolator_method = extrapolation.get_method(extrap_method)
R = R[-(ar_order + 1):, :, :].copy()
R_min = np.nanmin(R)
# determine the domain mask from non-finite values
domain_mask = np.logical_or.reduce(
[~np.isfinite(R[i, :]) for i in range(R.shape[0])])
# determine the precipitation threshold mask
if conditional:
MASK_thr = np.logical_and.reduce(
[R[i, :, :] >= R_thr for i in range(R.shape[0])])
else:
MASK_thr = None
# initialize the extrapolator
x_values, y_values = np.meshgrid(np.arange(R.shape[2]),
np.arange(R.shape[1]))
xy_coords = np.stack([x_values, y_values])
extrap_kwargs = extrap_kwargs.copy()
extrap_kwargs["xy_coords"] = xy_coords
extrap_kwargs["allow_nonfinite_values"] = True
# advect the previous precipitation fields to the same position with the
# most recent one (i.e. transform them into the Lagrangian coordinates)
res = list()
def f(R, i):
return extrapolator_method(R[i, :], V, ar_order - i, "min",
**extrap_kwargs)[-1]
for i in range(ar_order):
if not DASK_IMPORTED:
R[i, :, :] = f(R, i)
else:
res.append(dask.delayed(f)(R, i))
if DASK_IMPORTED:
num_workers_ = len(res) if num_workers > len(res) else num_workers
R = np.stack(
list(dask.compute(*res, num_workers=num_workers_)) + [R[-1, :, :]])
# replace non-finite values with the minimum value
R = R.copy()
for i in range(R.shape[0]):
R[i, ~np.isfinite(R[i, :])] = np.nanmin(R[i, :])
# compute the cascade decompositions of the input precipitation fields
R_d = []
for i in range(ar_order + 1):
R_ = decomp_method(
R[i, :, :],
filter,
mask=MASK_thr,
fft_method=fft,
output_domain=domain,
normalize=True,
compute_stats=True,
compact_output=True,
)
R_d.append(R_)
# rearrange the cascade levels into a four-dimensional array of shape
# (n_cascade_levels,ar_order+1,m,n) for the autoregressive model
R_c = nowcast_utils.stack_cascades(R_d,
n_cascade_levels,
convert_to_full_arrays=True)
# compute lag-l temporal autocorrelation coefficients for each cascade level
GAMMA = np.empty((n_cascade_levels, ar_order))
for i in range(n_cascade_levels):
if domain == "spatial":
GAMMA[i, :] = correlation.temporal_autocorrelation(R_c[i],
mask=MASK_thr)
else:
GAMMA[i, :] = correlation.temporal_autocorrelation(
R_c[i], domain="spectral", x_shape=R.shape[1:])
R_c = nowcast_utils.stack_cascades(R_d,
n_cascade_levels,
convert_to_full_arrays=False)
R_d = R_d[-1]
nowcast_utils.print_corrcoefs(GAMMA)
if ar_order == 2:
# adjust the lag-2 correlation coefficient to ensure that the AR(p)
# process is stationary
for i in range(n_cascade_levels):
GAMMA[i, 1] = autoregression.adjust_lag2_corrcoef2(
GAMMA[i, 0], GAMMA[i, 1])
# estimate the parameters of the AR(p) model from the autocorrelation
# coefficients
PHI = np.empty((n_cascade_levels, ar_order + 1))
for i in range(n_cascade_levels):
PHI[i, :] = autoregression.estimate_ar_params_yw(GAMMA[i, :])
nowcast_utils.print_ar_params(PHI)
# discard all except the p-1 last cascades because they are not needed for
# the AR(p) model
R_c = [R_c[i][-ar_order:] for i in range(n_cascade_levels)]
D = None
if probmatching_method == "mean":
mu_0 = np.mean(R[-1, :, :][R[-1, :, :] >= R_thr])
# compute precipitation mask and wet area ratio
MASK_p = R[-1, :, :] >= R_thr
war = 1.0 * np.sum(MASK_p) / (R.shape[1] * R.shape[2])
if measure_time:
init_time = time.time() - starttime_init
R = R[-1, :, :]
print("Starting nowcast computation.")
if measure_time:
starttime_mainloop = time.time()
R_f = []
# iterate each time step
for t in range(n_timesteps):
print("Computing nowcast for time step %d... " % (t + 1), end="")
sys.stdout.flush()
if measure_time:
starttime = time.time()
for i in range(n_cascade_levels):
R_c[i] = autoregression.iterate_ar_model(R_c[i], PHI[i, :])
R_d["cascade_levels"] = [
R_c[i][-1, :] for i in range(n_cascade_levels)
]
if domain == "spatial":
R_d["cascade_levels"] = np.stack(R_d["cascade_levels"])
R_c_ = recomp_method(R_d)
if domain == "spectral":
R_c_ = fft.irfft2(R_c_)
MASK = _compute_sprog_mask(R_c_, war)
R_c_[~MASK] = R_min
if probmatching_method == "cdf":
# adjust the CDF of the forecast to match the most recently
# observed precipitation field
R_c_ = probmatching.nonparam_match_empirical_cdf(R_c_, R)
elif probmatching_method == "mean":
mu_fct = np.mean(R_c_[MASK])
R_c_[MASK] = R_c_[MASK] - mu_fct + mu_0
R_c_[domain_mask] = np.nan
# advect the recomposed precipitation field to obtain the forecast for
# time step t
extrap_kwargs.update({
"displacement_prev": D,
"return_displacement": True
})
R_f_, D_ = extrapolator_method(R_c_, V, 1, **extrap_kwargs)
D = D_
R_f_ = R_f_[0]
R_f.append(R_f_)
if measure_time:
print("%.2f seconds." % (time.time() - starttime))
else:
print("done.")
if measure_time:
mainloop_time = time.time() - starttime_mainloop
R_f = np.stack(R_f)
if measure_time:
return R_f, init_time, mainloop_time
else:
return R_f
def worker(j):
if noise_method is not None:
# generate noise field
EPS = generate_noise(
pp, randstate=randgen_prec[j], fft_method=fft_objs[j], domain=domain
)
# decompose the noise field into a cascade
EPS = decomp_method(
EPS,
filter,
fft_method=fft_objs[j],
input_domain=domain,
output_domain=domain,
compute_stats=True,
normalize=True,
compact_output=True,
)
else:
EPS = None
# iterate the AR(p) model for each cascade level
for i in range(n_cascade_levels):
# normalize the noise cascade
if EPS is not None:
EPS_ = EPS["cascade_levels"][i]
EPS_ *= noise_std_coeffs[i]
else:
EPS_ = None
# apply AR(p) process to cascade level
if EPS is not None or vel_pert_method is not None:
R_c[j][i] = autoregression.iterate_ar_model(
R_c[j][i], PHI[i, :], eps=EPS_
)
else:
# use the deterministic AR(p) model computed above if
# perturbations are disabled
R_c[j][i] = R_m[i]
EPS = None
EPS_ = None
# compute the recomposed precipitation field(s) from the cascades
# obtained from the AR(p) model(s)
R_d[j]["cascade_levels"] = [
R_c[j][i][-1, :] for i in range(n_cascade_levels)
]
if domain == "spatial":
R_d[j]["cascade_levels"] = np.stack(R_d[j]["cascade_levels"])
R_f_new = recomp_method(R_d[j])
if domain == "spectral":
R_f_new = fft_objs[j].irfft2(R_f_new)
if mask_method is not None:
# apply the precipitation mask to prevent generation of new
# precipitation into areas where it was not originally
# observed
R_cmin = R_f_new.min()
if mask_method == "incremental":
R_f_new = R_cmin + (R_f_new - R_cmin) * MASK_prec[j]
MASK_prec_ = R_f_new > R_cmin
else:
MASK_prec_ = MASK_prec
# Set to min value outside of mask
R_f_new[~MASK_prec_] = R_cmin
if probmatching_method == "cdf":
# adjust the CDF of the forecast to match the most recently
# observed precipitation field
R_f_new = probmatching.nonparam_match_empirical_cdf(R_f_new, R)
elif probmatching_method == "mean":
MASK = R_f_new >= R_thr
mu_fct = np.mean(R_f_new[MASK])
R_f_new[MASK] = R_f_new[MASK] - mu_fct + mu_0
if mask_method == "incremental":
MASK_prec[j] = _compute_incremental_mask(
R_f_new >= R_thr, struct, mask_rim
)
R_f_new[domain_mask] = np.nan
R_f_out = []
extrap_kwargs_ = extrap_kwargs.copy()
V_pert = V
# advect the recomposed precipitation field to obtain the forecast for
# the current time step (or subtimesteps if non-integer time steps are
# given)
for t_sub in subtimesteps:
if t_sub > 0:
t_diff_prev_int = t_sub - int(t_sub)
if t_diff_prev_int > 0.0:
R_f_ip = (1.0 - t_diff_prev_int) * R_f_prev[
j
] + t_diff_prev_int * R_f_new
else:
R_f_ip = R_f_prev[j]
t_diff_prev = t_sub - t_prev[j]
t_total[j] += t_diff_prev
# compute the perturbed motion field
if vel_pert_method is not None:
V_pert = V + generate_vel_noise(vps[j], t_total[j] * timestep)
extrap_kwargs_["displacement_prev"] = D[j]
R_f_ep, D[j] = extrapolator_method(
R_f_ip,
V_pert,
[t_diff_prev],
**extrap_kwargs_,
)
R_f_out.append(R_f_ep[0])
t_prev[j] = t_sub
# advect the forecast field by one time step if no subtimesteps in the
# current interval were found
if not subtimesteps:
t_diff_prev = t + 1 - t_prev[j]
t_total[j] += t_diff_prev
# compute the perturbed motion field
if vel_pert_method is not None:
V_pert = V + generate_vel_noise(vps[j], t_total[j] * timestep)
extrap_kwargs_["displacement_prev"] = D[j]
_, D[j] = extrapolator_method(
None,
V_pert,
[t_diff_prev],
**extrap_kwargs_,
)
t_prev[j] = t + 1
R_f_prev[j] = R_f_new
return R_f_out
def worker(j):
if noise_method is not None:
# generate noise field
EPS = generate_noise(
pp, randstate=randgen_prec[j], fft_method=fft_objs[j], domain=domain
)
# decompose the noise field into a cascade
EPS = decomp_method(
EPS,
filter,
fft_method=fft_objs[j],
input_domain=domain,
output_domain=domain,
compute_stats=True,
normalize=True,
compact_output=True,
)
else:
EPS = None
# iterate the AR(p) model for each cascade level
for i in range(n_cascade_levels):
# normalize the noise cascade
if EPS is not None:
EPS_ = EPS["cascade_levels"][i]
EPS_ *= noise_std_coeffs[i]
else:
EPS_ = None
# apply AR(p) process to cascade level
if EPS is not None or vel_pert_method is not None:
R_c[j][i] = autoregression.iterate_ar_model(
R_c[j][i], PHI[i, :], eps=EPS_
)
else:
# use the deterministic AR(p) model computed above if
# perturbations are disabled
R_c[j][i] = R_m[i]
EPS = None
EPS_ = None
# compute the recomposed precipitation field(s) from the cascades
# obtained from the AR(p) model(s)
R_d[j]["cascade_levels"] = [
R_c[j][i][-1, :] for i in range(n_cascade_levels)
]
if domain == "spatial":
R_d[j]["cascade_levels"] = np.stack(R_d[j]["cascade_levels"])
R_c_ = recomp_method(R_d[j])
if domain == "spectral":
R_c_ = fft_objs[j].irfft2(R_c_)
if mask_method is not None:
# apply the precipitation mask to prevent generation of new
# precipitation into areas where it was not originally
# observed
R_cmin = R_c_.min()
if mask_method == "incremental":
R_c_ = R_cmin + (R_c_ - R_cmin) * MASK_prec[j]
MASK_prec_ = R_c_ > R_cmin
else:
MASK_prec_ = MASK_prec
# Set to min value outside of mask
R_c_[~MASK_prec_] = R_cmin
if probmatching_method == "cdf":
# adjust the CDF of the forecast to match the most recently
# observed precipitation field
R_c_ = probmatching.nonparam_match_empirical_cdf(R_c_, R)
elif probmatching_method == "mean":
MASK = R_c_ >= R_thr
mu_fct = np.mean(R_c_[MASK])
R_c_[MASK] = R_c_[MASK] - mu_fct + mu_0
if mask_method == "incremental":
MASK_prec[j] = _compute_incremental_mask(
R_c_ >= R_thr, struct, mask_rim
)
# compute the perturbed motion field
if vel_pert_method is not None:
V_ = V + generate_vel_noise(vps[j], (t + 1) * timestep)
else:
V_ = V
R_c_[domain_mask] = np.nan
# advect the recomposed precipitation field to obtain the forecast
# for time step t
extrap_kwargs_ = extrap_kwargs.copy()
extrap_kwargs_["displacement_prev"] = D[j]
R_f_, D_ = extrapolator_method(R_c_, V_, 1, **extrap_kwargs_)
D[j] = D_
R_f_ = R_f_[0]
return R_f_
def plot_raincast_det(start_datetime, end_datetime, analysis_time,
raincast_det_path, radar_path):
raincast_valid_times = []
with Dataset(raincast_det_path) as cur_nc:
raincast = cur_nc.variables['forecast'][:]
latitude = cur_nc.variables['latitude'][:]
longitude = cur_nc.variables['longitude'][:]
data_mask = cur_nc.variables['data_mask'][:]
time = cur_nc.variables['time'][:]
calendar = 'gregorian'
units = 'seconds since 1970-01-01 00:0:0'
for t in range(len(time)):
raincast_valid_times.append(
num2date(time[t], units=units, calendar=calendar))
radar_valid_times = []
with Dataset(radar_path) as cur_nc:
radar = cur_nc.variables['forecast'][:]
latitude = cur_nc.variables['latitude'][:]
longitude = cur_nc.variables['longitude'][:]
data_mask = cur_nc.variables['data_mask'][:]
time = cur_nc.variables['time'][:]
calendar = 'gregorian'
units = 'seconds since 1970-01-01 00:0:0'
for t in range(len(time)):
radar_valid_times.append(
num2date(time[t], units=units, calendar=calendar))
valid_time = start_datetime
while valid_time <= end_datetime:
print('raincast_det: start processing {}'.format(valid_time))
t_index0 = raincast_valid_times.index(valid_time)
raincast_data = raincast[t_index0, :, :]
t_index1 = radar_valid_times.index(valid_time)
radar_data = radar[t_index1, :, :]
raincast_data[numpy.isnan(raincast_data)] = 0.0
radar_data[numpy.isnan(radar_data)] = 0.0
raincast_data = probmatching.nonparam_match_empirical_cdf(
raincast_data, radar_data)
longitude = longitude % 360.0
generate_plot(latitude,
longitude,
raincast_data,
'raincast',
analysis_time,
valid_time,
'{model_name}_{valid_time}.png'.format(
model_name='raincast',
valid_time=valid_time.strftime('%Y%m%d%H%M')),
llcrnrlon=172,
urcrnrlon=180,
llcrnrlat=-40.0,
urcrnrlat=-35.0)
generate_plot(latitude,
longitude,
radar_data,
'radar',
analysis_time,
valid_time,
'{model_name}_{valid_time}.png'.format(
model_name='radar',
valid_time=valid_time.strftime('%Y%m%d%H%M')),
llcrnrlon=172,
urcrnrlon=180,
llcrnrlat=-40.0,
urcrnrlat=-35.0)
valid_time += timedelta(seconds=600.0)
def worker(j):
if noise_method is not None:
# generate noise field
EPS = generate_noise(pp, randstate=randgen_prec[j],
fft_method=fft_objs[j])
# decompose the noise field into a cascade
EPS = decomp_method(EPS, filter, fft_method=fft_objs[j])
else:
EPS = None
# iterate the AR(p) model for each cascade level
for i in range(n_cascade_levels):
# normalize the noise cascade
if EPS is not None:
EPS_ = (EPS["cascade_levels"][i, :, :] - EPS["means"][i]) / EPS["stds"][i]
EPS_ *= noise_std_coeffs[i]
else:
EPS_ = None
# apply AR(p) process to cascade level
if EPS is not None or vel_pert_method is not None:
R_c[j, i, :, :, :] = \
autoregression.iterate_ar_model(R_c[j, i, :, :, :],
PHI[i, :], EPS=EPS_)
else:
# use the deterministic AR(p) model computed above if
# perturbations are disabled
R_c[j, i, :, :, :] = R_m[i, :, :, :]
EPS = None
EPS_ = None
# compute the recomposed precipitation field(s) from the cascades
# obtained from the AR(p) model(s)
R_c_ = nowcast_utils.recompose_cascade(R_c[j, :, -1, :, :], mu, sigma)
if mask_method is not None:
# apply the precipitation mask to prevent generation of new
# precipitation into areas where it was not originally
# observed
R_cmin = R_c_.min()
if mask_method == "incremental":
R_c_ = R_cmin + (R_c_ - R_cmin) * MASK_prec[j]
MASK_prec_ = R_c_ > R_cmin
else:
MASK_prec_ = MASK_prec
# Set to min value outside of mask
R_c_[~MASK_prec_] = R_cmin
if probmatching_method == "cdf":
# adjust the CDF of the forecast to match the most recently
# observed precipitation field
R_c_ = probmatching.nonparam_match_empirical_cdf(R_c_, R)
elif probmatching_method == "mean":
MASK = R_c_ >= R_thr
mu_fct = np.mean(R_c_[MASK])
R_c_[MASK] = R_c_[MASK] - mu_fct + mu_0
if mask_method == "incremental":
MASK_prec[j] = _compute_incremental_mask(R_c_ >= R_thr, struct, mask_rim)
# compute the perturbed motion field
if vel_pert_method is not None:
V_ = V + generate_vel_noise(vps[j], (t + 1) * timestep)
else:
V_ = V
# advect the recomposed precipitation field to obtain the forecast
# for time step t
extrap_kwargs.update({"D_prev": D[j], "return_displacement": True})
R_f_, D_ = extrapolator_method(R_c_, V_, 1, **extrap_kwargs)
D[j] = D_
R_f_ = R_f_[0]
return R_f_