def test_wrong_inputs(interp_method): coord = np.random.rand(10, 2) input_array = np.random.rand(10, 2) xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10) interp = get_method(interp_method) # nan in the input values with pytest.raises(ValueError): input_with_nans = input_array.copy() input_with_nans[0, 0] = np.nan interp(coord, input_with_nans, xgrid, ygrid) # nan in the input coordinates with pytest.raises(ValueError): coord_with_nans = coord.copy() coord_with_nans[0, 0] = np.nan interp(coord_with_nans, input_array, xgrid, ygrid) # too many dimensions in the input values with pytest.raises(ValueError): interp(coord, np.random.rand(10, 2, 1), xgrid, ygrid) # wrong dimension size in the input coordinates with pytest.raises(ValueError): interp(np.random.rand(10, 1), input_array, xgrid, ygrid) # wrong number of dimensions in the input coordinates with pytest.raises(ValueError): interp(np.random.rand(10, 2, 1), input_array, xgrid, ygrid) # wrong number of coordinates with pytest.raises(ValueError): interp(np.random.rand(9, 2), input_array, xgrid, ygrid)
def test_uniform_input(interp_method): coord = np.random.rand(10, 2) xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10) interp = get_method(interp_method) # same value across all variables input_array = np.ones((10, 2)) output = interp(coord, input_array, xgrid, ygrid) assert np.isfinite(output).all() assert output.max() == output.min() == input_array.ravel()[0]
def test_one_sample_input(interp_method): coord = np.random.rand(1, 2) input_array = np.array([1, 2])[None, :] xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10) interp = get_method(interp_method) # one sample returns uniform grids output = interp(coord, input_array, xgrid, ygrid) assert np.isfinite(output).all() assert output[0, ...].max() == output[0, ...].min() == 1 assert output[1, ...].max() == output[1, ...].min() == 2
def test_interp_univariate(interp_method): coord = np.random.rand(10, 2) input_array = np.random.rand(10) xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10) interp = get_method(interp_method) output = interp(coord, input_array, xgrid, ygrid) assert isinstance(output, np.ndarray) assert output.ndim == 2 assert output.shape == (ygrid.size, xgrid.size) assert np.isfinite(output).all()
def test_idwinterp2d_kNone(): coord = np.random.rand(10, 2) input_array = np.random.rand(10, 2) xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10) interp = get_method("idwinterp2d") output = interp(coord, input_array, xgrid, ygrid, k=None) assert isinstance(output, np.ndarray) assert output.ndim == 3 assert output.shape[0] == 2 assert output.shape[1:] == (ygrid.size, xgrid.size) assert np.isfinite(output).all()
def forecast( vil, velocity, n_timesteps, rainrate=None, n_cascade_levels=8, extrap_method="semilagrangian", ar_order=2, ar_window_radius=50, r_vil_window_radius=3, fft_method="numpy", apply_rainrate_mask=True, num_workers=1, extrap_kwargs=None, filter_kwargs=None, measure_time=False, ): """Generate a nowcast by using the autoregressive nowcasting using VIL (ANVIL) method. ANVIL is built on top of an extrapolation-based nowcast. The key features are: 1) Growth and decay: implemented by using a cascade decomposition and a multiscale autoregressive integrated ARI(p,1) model. Instead of the original time series, the ARI model is applied to the differenced one corresponding to time derivatives. 2) Originally designed for using integrated liquid (VIL) as the input data. In this case, the rain rate (R) is obtained from VIL via an empirical relation. This implementation is more general so that the input can be any two-dimensional precipitation field. 3) The parameters of the ARI model and the R(VIL) relation are allowed to be spatially variable. The estimation is done using a moving window. Parameters ---------- vil : array_like Array of shape (ar_order+2,m,n) containing the input fields ordered by timestamp from oldest to newest. The inputs are expected to contain VIL or rain rate. The time steps between the inputs are assumed to be regular. velocity : array_like Array of shape (2,m,n) containing the x- and y-components of the advection field. The velocities are assumed to represent one time step between the inputs. All values are required to be finite. n_timesteps : int Number of time steps to forecast. rainrate : array_like Array of shape (m,n) containing the most recently observed rain rate field. If set to None, no R(VIL) conversion is done and the outputs are in the same units as the inputs. n_cascade_levels : int, optional The number of cascade levels to use. extrap_method : str, optional Name of the extrapolation method to use. See the documentation of pysteps.extrapolation.interface. ar_order : int, optional The order of the autoregressive model to use. The recommended values are 1 or 2. Using a higher-order model is strongly discouraged because the stationarity of the AR process cannot be guaranteed. ar_window_radius : int, optional The radius of the window to use for determining the parameters of the autoregressive model. Set to None to disable localization. r_vil_window_radius : int, optional The radius of the window to use for determining the R(VIL) relation. Applicable if rainrate is not None. fft_method : str, optional A string defining the FFT method to use (see utils.fft.get_method). Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed, the recommended method is 'pyfftw'. apply_rainrate_mask : bool Apply mask to prevent producing precipitation to areas where it was not originally observed. Defaults to True. Disabling this may improve some verification metrics but increases the number of false alarms. Applicable if rainrate is None. num_workers : int, optional The number of workers to use for parallel computation. Applicable if dask is installed or pyFFTW is used for computing the FFT. When num_workers>1, it is advisable to disable OpenMP by setting the environment variable OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous threads. extrap_kwargs : dict, optional Optional dictionary containing keyword arguments for the extrapolation method. See the documentation of pysteps.extrapolation. filter_kwargs : dict, optional Optional dictionary containing keyword arguments for the filter method. See the documentation of pysteps.cascade.bandpass_filters.py. measure_time : bool, optional If True, measure, print and return the computation time. Returns ------- out : ndarray A three-dimensional array of shape (n_timesteps,m,n) containing a time series of forecast precipitation fields. The time series starts from t0+timestep, where timestep is taken from the input VIL/rain rate fields. If measure_time is True, the return value is a three-element tuple containing the nowcast array, the initialization time of the nowcast generator and the time used in the main loop (seconds). References ---------- :cite:`PCLH2020` """ if len(vil.shape) != 3: raise ValueError( "vil.shape = %s, but a three-dimensional array expected" % str(vil.shape) ) if rainrate is not None: if len(rainrate.shape) != 2: raise ValueError( "rainrate.shape = %s, but a two-dimensional array expected" % str(rainrate.shape) ) if vil.shape[0] != ar_order + 2: raise ValueError( "vil.shape[0] = %d, but vil.shape[0] = ar_order + 2 = %d required" % (vil.shape[0], ar_order + 2) ) if len(velocity.shape) != 3: raise ValueError( "velocity.shape = %s, but a three-dimensional array expected" % str(velocity.shape) ) if extrap_kwargs is None: extrap_kwargs = dict() if filter_kwargs is None: filter_kwargs = dict() print("Computing ANVIL nowcast:") print("------------------------") print("") print("Inputs:") print("-------") print("input dimensions: %dx%d" % (vil.shape[1], vil.shape[2])) print("") print("Methods:") print("--------") print("extrapolation: %s" % extrap_method) print("FFT: %s" % fft_method) print("") print("Parameters:") print("-----------") print("number of time steps: %d" % n_timesteps) print("parallel threads: %d" % num_workers) print("number of cascade levels: %d" % n_cascade_levels) print("order of the ARI(p,1) model: %d" % ar_order) if type(ar_window_radius) == int: print("ARI(p,1) window radius: %d" % ar_window_radius) else: print("ARI(p,1) window radius: none") print("R(VIL) window radius: %d" % r_vil_window_radius) if measure_time: starttime_init = time.time() m, n = vil.shape[1:] vil = vil.copy() if rainrate is None and apply_rainrate_mask: rainrate_mask = vil[-1, :] < 0.1 if rainrate is not None: # determine the coefficients fields of the relation R=a*VIL+b by # localized linear regression r_vil_a, r_vil_b = _r_vil_regression(vil[-1, :], rainrate, r_vil_window_radius) # transform the input fields to Lagrangian coordinates by extrapolation extrapolator = extrapolation.get_method(extrap_method) res = list() def worker(vil, i): return ( i, extrapolator( vil[i, :], velocity, vil.shape[0] - 1 - i, allow_nonfinite_values=True, **extrap_kwargs, )[-1], ) for i in range(vil.shape[0] - 1): if not DASK_IMPORTED or num_workers == 1: vil[i, :, :] = worker(vil, i)[1] else: res.append(dask.delayed(worker)(vil, i)) if DASK_IMPORTED and num_workers > 1: num_workers_ = len(res) if num_workers > len(res) else num_workers vil_e = dask.compute(*res, num_workers=num_workers_) for i in range(len(vil_e)): vil[vil_e[i][0], :] = vil_e[i][1] # compute the final mask as the intersection of the masks of the advected # fields mask = np.isfinite(vil[0, :]) for i in range(1, vil.shape[0]): mask = np.logical_and(mask, np.isfinite(vil[i, :])) if rainrate is None and apply_rainrate_mask: rainrate_mask = np.logical_and(rainrate_mask, mask) # apply cascade decomposition to the advected input fields bp_filter_method = cascade.get_method("gaussian") bp_filter = bp_filter_method((m, n), n_cascade_levels, **filter_kwargs) fft = utils.get_method(fft_method, shape=vil.shape[1:], n_threads=num_workers) decomp_method, recomp_method = cascade.get_method("fft") vil_dec = np.empty((n_cascade_levels, vil.shape[0], m, n)) for i in range(vil.shape[0]): vil_ = vil[i, :].copy() vil_[~np.isfinite(vil_)] = 0.0 vil_dec_i = decomp_method(vil_, bp_filter, fft_method=fft) for j in range(n_cascade_levels): vil_dec[j, i, :] = vil_dec_i["cascade_levels"][j, :] # compute time-lagged correlation coefficients for the cascade levels of # the advected and differenced input fields gamma = np.empty((n_cascade_levels, ar_order, m, n)) for i in range(n_cascade_levels): vil_diff = np.diff(vil_dec[i, :], axis=0) vil_diff[~np.isfinite(vil_diff)] = 0.0 for j in range(ar_order): gamma[i, j, :] = _moving_window_corrcoef( vil_diff[-1, :], vil_diff[-(j + 2), :], ar_window_radius ) if ar_order == 2: # if the order of the ARI model is 2, adjust the correlation coefficients # so that the resulting process is stationary for i in range(n_cascade_levels): gamma[i, 1, :] = autoregression.adjust_lag2_corrcoef2( gamma[i, 0, :], gamma[i, 1, :] ) # estimate the parameters of the ARI models phi = [] for i in range(n_cascade_levels): if ar_order > 2: phi_ = autoregression.estimate_ar_params_yw_localized(gamma[i, :], d=1) elif ar_order == 2: phi_ = _estimate_ar2_params(gamma[i, :]) else: phi_ = _estimate_ar1_params(gamma[i, :]) phi.append(phi_) vil_dec = vil_dec[:, -(ar_order + 1) :, :] if measure_time: init_time = time.time() - starttime_init print("Starting nowcast computation.") if measure_time: starttime_mainloop = time.time() r_f = [] dp = None for t in range(n_timesteps): print("Computing nowcast for time step %d... " % (t + 1), end="", flush=True) if measure_time: starttime = time.time() # iterate the ARI models for each cascade level for i in range(n_cascade_levels): vil_dec[i, :] = autoregression.iterate_ar_model(vil_dec[i, :], phi[i]) # recompose the cascade to obtain the forecast field vil_dec_dict = {} vil_dec_dict["cascade_levels"] = vil_dec[:, -1, :] vil_dec_dict["domain"] = "spatial" vil_dec_dict["normalized"] = False vil_f = recomp_method(vil_dec_dict) vil_f[~mask] = np.nan if rainrate is not None: # convert VIL to rain rate r_f_ = r_vil_a * vil_f + r_vil_b else: r_f_ = vil_f if apply_rainrate_mask: r_f_[rainrate_mask] = 0.0 r_f_[r_f_ < 0.0] = 0.0 # extrapolate to the current nowcast lead time extrap_kwargs.update( { "displacement_prev": dp, "return_displacement": True, "allow_nonfinite_values": True, } ) r_f_, dp = extrapolator(r_f_, velocity, 1, **extrap_kwargs) if measure_time: print("%.2f seconds." % (time.time() - starttime)) else: print("done.") r_f.append(r_f_[-1]) if measure_time: mainloop_time = time.time() - starttime_mainloop if measure_time: return np.stack(r_f), init_time, mainloop_time else: return np.stack(r_f)
def dense_lucaskanade( input_images, lk_kwargs=None, fd_method="ShiTomasi", fd_kwargs=None, interp_method="rbfinterp2d", interp_kwargs=None, dense=True, nr_std_outlier=3, k_outlier=30, size_opening=3, decl_scale=20, verbose=False, ): """Run the Lucas-Kanade optical flow routine and interpolate the motion vectors. .. _OpenCV: https://opencv.org/ .. _`Lucas-Kanade`:\ https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323 .. _MaskedArray:\ https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray .. _ndarray:\ https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html Interface to the OpenCV_ implementation of the local `Lucas-Kanade`_ optical flow method applied in combination to a feature detection routine. The sparse motion vectors are finally interpolated to return the whole motion field. Parameters ---------- input_images : ndarray_ or MaskedArray_ Array of shape (T, m, n) containing a sequence of *T* two-dimensional input images of shape (m, n). The indexing order in **input_images** is assumed to be (time, latitude, longitude). *T* = 2 is the minimum required number of images. With *T* > 2, all the resulting sparse vectors are pooled together for the final interpolation on a regular grid. In case of ndarray_, invalid values (Nans or infs) are masked, otherwise the mask of the MaskedArray_ is used. Such mask defines a region where features are not detected for the tracking algorithm. lk_kwargs : dict, optional Optional dictionary containing keyword arguments for the `Lucas-Kanade`_ features tracking algorithm. See the documentation of :py:func:`pysteps.motion.lucaskanade.track_features`. fd_method : {"ShiTomasi", "blob"}, optional Name of the feature detection routine. See feature detection methods in :py:mod:`pysteps.utils.images`. fd_kwargs : dict, optional Optional dictionary containing keyword arguments for the features detection algorithm. See the documentation of :py:mod:`pysteps.utils.images`. interp_method : {"rbfinterp2d"}, optional Name of the interpolation method to use. See interpolation methods in :py:mod:`pysteps.utils.interpolate`. interp_kwargs : dict, optional Optional dictionary containing keyword arguments for the interpolation algorithm. See the documentation of :py:mod:`pysteps.utils.interpolate`. dense : bool, optional If True, return the three-dimensional array (2, m, n) containing the dense x- and y-components of the motion field. If False, return the sparse motion vectors as 2-D **xy** and **uv** arrays, where **xy** defines the vector positions, **uv** defines the x and y direction components of the vectors. nr_std_outlier : int, optional Maximum acceptable deviation from the mean in terms of number of standard deviations. Any sparse vector with a deviation larger than this threshold is flagged as outlier and excluded from the interpolation. See the documentation of :py:func:`pysteps.utils.cleansing.detect_outliers`. k_outlier : int or None, optional The number of nearest neighbours used to localize the outlier detection. If set to None, it employs all the data points (global detection). See the documentation of :py:func:`pysteps.utils.cleansing.detect_outliers`. size_opening : int, optional The size of the structuring element kernel in pixels. This is used to perform a binary morphological opening on the input fields in order to filter isolated echoes due to clutter. If set to zero, the filtering is not perfomed. See the documentation of :py:func:`pysteps.utils.images.morph_opening`. decl_scale : int, optional The scale declustering parameter in pixels used to reduce the number of redundant sparse vectors before the interpolation. Sparse vectors within this declustering scale are averaged together. If set to less than 2 pixels, the declustering is not perfomed. See the documentation of :py:func:`pysteps.utils.cleansing.decluster`. verbose : bool, optional If set to True, print some information about the program. Returns ------- out : ndarray_ or tuple If **dense=True** (the default), return the advection field having shape (2, m, n), where out[0, :, :] contains the x-components of the motion vectors and out[1, :, :] contains the y-components. The velocities are in units of pixels / timestep, where timestep is the time difference between the two input images. Return a zero motion field of shape (2, m, n) when no motion is detected. If **dense=False**, it returns a tuple containing the 2-dimensional arrays **xy** and **uv**, where x, y define the vector locations, u, v define the x and y direction components of the vectors. Return two empty arrays when no motion is detected. See also -------- pysteps.motion.lucaskanade.track_features References ---------- Bouguet, J.-Y.: Pyramidal implementation of the affine Lucas Kanade feature tracker description of the algorithm, Intel Corp., 5, 4, https://doi.org/10.1109/HPDC.2004.1323531, 2001 Lucas, B. D. and Kanade, T.: An iterative image registration technique with an application to stereo vision, in: Proceedings of the 1981 DARPA Imaging Understanding Workshop, pp. 121–130, 1981. """ input_images = input_images.copy() if verbose: print("Computing the motion field with the Lucas-Kanade method.") t0 = time.time() nr_fields = input_images.shape[0] domain_size = (input_images.shape[1], input_images.shape[2]) feature_detection_method = utils.get_method(fd_method) interpolation_method = utils.get_method(interp_method) if fd_kwargs is None: fd_kwargs = dict() if lk_kwargs is None: lk_kwargs = dict() if interp_kwargs is None: interp_kwargs = dict() xy = np.empty(shape=(0, 2)) uv = np.empty(shape=(0, 2)) for n in range(nr_fields - 1): # extract consecutive images prvs_img = input_images[n, :, :].copy() next_img = input_images[n + 1, :, :].copy() # Check if a MaskedArray is used. If not, mask the ndarray if not isinstance(prvs_img, MaskedArray): prvs_img = np.ma.masked_invalid(prvs_img) np.ma.set_fill_value(prvs_img, prvs_img.min()) if not isinstance(next_img, MaskedArray): next_img = np.ma.masked_invalid(next_img) np.ma.set_fill_value(next_img, next_img.min()) # remove small noise with a morphological operator (opening) if size_opening > 0: prvs_img = morph_opening(prvs_img, prvs_img.min(), size_opening) next_img = morph_opening(next_img, next_img.min(), size_opening) # features detection points = feature_detection_method(prvs_img, **fd_kwargs).astype(np.float32) # skip loop if no features to track if points.shape[0] == 0: continue # get sparse u, v vectors with Lucas-Kanade tracking xy_, uv_ = track_features(prvs_img, next_img, points, **lk_kwargs) # skip loop if no vectors if xy_.shape[0] == 0: continue # stack vectors xy = np.append(xy, xy_, axis=0) uv = np.append(uv, uv_, axis=0) # return zero motion field is no sparse vectors are found if xy.shape[0] == 0: if dense: return np.zeros((2, domain_size[0], domain_size[1])) else: return xy, uv # detect and remove outliers outliers = detect_outliers(uv, nr_std_outlier, xy, k_outlier, verbose) xy = xy[~outliers, :] uv = uv[~outliers, :] if verbose: print("--- LK found %i sparse vectors ---" % xy.shape[0]) # return sparse vectors if required if not dense: return xy, uv # decluster sparse motion vectors if decl_scale > 1: xy, uv = decluster(xy, uv, decl_scale, 1, verbose) # return zero motion field if no sparse vectors are left for interpolation if xy.shape[0] == 0: return np.zeros((2, domain_size[0], domain_size[1])) # interpolation xgrid = np.arange(domain_size[1]) ygrid = np.arange(domain_size[0]) UV = interpolation_method(xy, uv, xgrid, ygrid, **interp_kwargs) if verbose: print("--- total time: %.2f seconds ---" % (time.time() - t0)) return UV
def decomposition_fft(field, bp_filter, **kwargs): """Decompose a two-dimensional input field into multiple spatial scales by using the Fast Fourier Transform (FFT) and a set of bandpass filters. Parameters ---------- field : array_like Two-dimensional array containing the input field. All values are required to be finite. bp_filter : dict A filter returned by a method implemented in :py:mod:`pysteps.cascade.bandpass_filters`. Other Parameters ---------------- fft_method : str or tuple A string or a (function,kwargs) tuple defining the FFT method to use (see :py:func:`pysteps.utils.interface.get_method`). Defaults to "numpy". This option is not used if input_domain and output_domain are both set to "spectral". normalize : bool If True, normalize the cascade levels to zero mean and unit variance. Requires that compute_stats is True. Implies that compute_stats is True. Defaults to False. mask : array_like Optional mask to use for computing the statistics for the cascade levels. Pixels with mask==False are excluded from the computations. This option is not used if output domain is "spectral". input_domain : {"spatial", "spectral"} The domain of the input field. If "spectral", the input is assumed to be in the spectral domain. Defaults to "spatial". output_domain : {"spatial", "spectral"} If "spatial", the output cascade levels are transformed back to the spatial domain by using the inverse FFT. If "spectral", the cascade is kept in the spectral domain. Defaults to "spatial". compute_stats : bool If True, the output dictionary contains the keys "means" and "stds" for the mean and standard deviation of each output cascade level. Defaults to False. compact_output : bool Applicable if output_domain is "spectral". If set to True, only the parts of the Fourier spectrum with non-negligible filter weights are stored. Defaults to False. Returns ------- out : ndarray A dictionary described in the module documentation. The number of cascade levels is determined from the filter (see :py:mod:`pysteps.cascade.bandpass_filters`). """ fft = kwargs.get("fft_method", "numpy") if isinstance(fft, str): fft = utils.get_method(fft, shape=field.shape) normalize = kwargs.get("normalize", False) mask = kwargs.get("mask", None) input_domain = kwargs.get("input_domain", "spatial") output_domain = kwargs.get("output_domain", "spatial") compute_stats = kwargs.get("compute_stats", False) compact_output = kwargs.get("compact_output", False) if normalize and not compute_stats: compute_stats = True if len(field.shape) != 2: raise ValueError("The input is not two-dimensional array") if mask is not None and mask.shape != field.shape: raise ValueError("Dimension mismatch between field and mask:" + "field.shape=" + str(field.shape) + ",mask.shape" + str(mask.shape)) if field.shape[0] != bp_filter["weights_2d"].shape[1]: raise ValueError("dimension mismatch between field and bp_filter: " + "field.shape[0]=%d , " % field.shape[0] + "bp_filter['weights_2d'].shape[1]" "=%d" % bp_filter["weights_2d"].shape[1]) if input_domain == "spatial" and \ int(field.shape[1] / 2) + 1 != bp_filter["weights_2d"].shape[2]: raise ValueError("Dimension mismatch between field and bp_filter: " "int(field.shape[1]/2)+1=%d , " % (int(field.shape[1] / 2) + 1) + "bp_filter['weights_2d'].shape[2]" "=%d" % bp_filter["weights_2d"].shape[2]) if input_domain == "spectral" and \ field.shape[1] != bp_filter["weights_2d"].shape[2]: raise ValueError("Dimension mismatch between field and bp_filter: " "field.shape[1]=%d , " % (field.shape[1] + 1) + "bp_filter['weights_2d'].shape[2]" "=%d" % bp_filter["weights_2d"].shape[2]) if output_domain != "spectral": compact_output = False if np.any(~np.isfinite(field)): raise ValueError("field contains non-finite values") result = {} means = [] stds = [] if input_domain == "spatial": field_fft = fft.rfft2(field) else: field_fft = field if output_domain == "spectral" and compact_output: weight_masks = [] field_decomp = [] for k in range(len(bp_filter["weights_1d"])): field_ = field_fft * bp_filter["weights_2d"][k, :, :] if output_domain == "spatial" or (compute_stats and mask is not None): field__ = fft.irfft2(field_) else: field__ = field_ if compute_stats: if output_domain == "spatial" or (compute_stats and mask is not None): if mask is not None: masked_field = field__[mask] else: masked_field = field__ mean = np.mean(masked_field) std = np.std(masked_field) else: mean = utils.spectral.mean(field_, bp_filter["shape"]) std = utils.spectral.std(field_, bp_filter["shape"]) means.append(mean) stds.append(std) if output_domain == "spatial": field_ = field__ if normalize: field_ = (field_ - mean) / std if output_domain == "spectral" and compact_output: weight_mask = bp_filter["weights_2d"][k, :, :] > 1e-12 field_ = field_[weight_mask] weight_masks.append(weight_mask) field_decomp.append(field_) result["domain"] = output_domain result["normalized"] = normalize result["compact_output"] = compact_output if output_domain == "spatial" or not compact_output: field_decomp = np.stack(field_decomp) result["cascade_levels"] = field_decomp if output_domain == "spectral" and compact_output: result["weight_masks"] = np.stack(weight_masks) if compute_stats: result["means"] = means result["stds"] = stds return result
def forecast( R, V, n_timesteps, n_cascade_levels=6, R_thr=None, extrap_method="semilagrangian", decomp_method="fft", bandpass_filter_method="gaussian", ar_order=2, conditional=False, probmatching_method="cdf", num_workers=1, fft_method="numpy", domain="spatial", extrap_kwargs=None, filter_kwargs=None, measure_time=False, ): """Generate a nowcast by using the Spectral Prognosis (S-PROG) method. Parameters ---------- R : array-like Array of shape (ar_order+1,m,n) containing the input precipitation fields ordered by timestamp from oldest to newest. The time steps between the inputs are assumed to be regular. V : array-like Array of shape (2,m,n) containing the x- and y-components of the advection field. The velocities are assumed to represent one time step between the inputs. All values are required to be finite. n_timesteps : int Number of time steps to forecast. n_cascade_levels : int, optional The number of cascade levels to use. R_thr : float The threshold value for minimum observable precipitation intensity. extrap_method : str, optional Name of the extrapolation method to use. See the documentation of pysteps.extrapolation.interface. decomp_method : {'fft'}, optional Name of the cascade decomposition method to use. See the documentation of pysteps.cascade.interface. bandpass_filter_method : {'gaussian', 'uniform'}, optional Name of the bandpass filter method to use with the cascade decomposition. See the documentation of pysteps.cascade.interface. ar_order : int, optional The order of the autoregressive model to use. Must be >= 1. conditional : bool, optional If set to True, compute the statistics of the precipitation field conditionally by excluding pixels where the values are below the threshold R_thr. probmatching_method : {'cdf','mean',None}, optional Method for matching the conditional statistics of the forecast field (areas with precipitation intensity above the threshold R_thr) with those of the most recently observed one. 'cdf'=map the forecast CDF to the observed one, 'mean'=adjust only the mean value, None=no matching applied. num_workers : int, optional The number of workers to use for parallel computation. Applicable if dask is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it is advisable to disable OpenMP by setting the environment variable OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous threads. fft_method : str, optional A string defining the FFT method to use (see utils.fft.get_method). Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed, the recommended method is 'pyfftw'. domain : {"spatial", "spectral"} If "spatial", all computations are done in the spatial domain (the classical S-PROG model). If "spectral", the AR(2) models are applied directly in the spectral domain to reduce memory footprint and improve performance :cite:`PCH2019a`. extrap_kwargs : dict, optional Optional dictionary containing keyword arguments for the extrapolation method. See the documentation of pysteps.extrapolation. filter_kwargs : dict, optional Optional dictionary containing keyword arguments for the filter method. See the documentation of pysteps.cascade.bandpass_filters.py. measure_time : bool If set to True, measure, print and return the computation time. Returns ------- out : ndarray A three-dimensional array of shape (n_timesteps,m,n) containing a time series of forecast precipitation fields. The time series starts from t0+timestep, where timestep is taken from the input precipitation fields R. If measure_time is True, the return value is a three-element tuple containing the nowcast array, the initialization time of the nowcast generator and the time used in the main loop (seconds). See also -------- pysteps.extrapolation.interface, pysteps.cascade.interface References ---------- :cite:`Seed2003`, :cite:`PCH2019a` """ _check_inputs(R, V, ar_order) if extrap_kwargs is None: extrap_kwargs = dict() if filter_kwargs is None: filter_kwargs = dict() if np.any(~np.isfinite(V)): raise ValueError("V contains non-finite values") print("Computing S-PROG nowcast:") print("-------------------------") print("") print("Inputs:") print("-------") print("input dimensions: %dx%d" % (R.shape[1], R.shape[2])) print("") print("Methods:") print("--------") print("extrapolation: %s" % extrap_method) print("bandpass filter: %s" % bandpass_filter_method) print("decomposition: %s" % decomp_method) print("conditional statistics: %s" % ("yes" if conditional else "no")) print("probability matching: %s" % probmatching_method) print("FFT method: %s" % fft_method) print("domain: %s" % domain) print("") print("Parameters:") print("-----------") print("number of time steps: %d" % n_timesteps) print("parallel threads: %d" % num_workers) print("number of cascade levels: %d" % n_cascade_levels) print("order of the AR(p) model: %d" % ar_order) print("precip. intensity threshold: %g" % R_thr) if measure_time: starttime_init = time.time() fft = utils.get_method(fft_method, shape=R.shape[1:], n_threads=num_workers) M, N = R.shape[1:] # initialize the band-pass filter filter_method = cascade.get_method(bandpass_filter_method) filter = filter_method((M, N), n_cascade_levels, **filter_kwargs) decomp_method, recomp_method = cascade.get_method(decomp_method) extrapolator_method = extrapolation.get_method(extrap_method) R = R[-(ar_order + 1):, :, :].copy() R_min = np.nanmin(R) # determine the domain mask from non-finite values domain_mask = np.logical_or.reduce( [~np.isfinite(R[i, :]) for i in range(R.shape[0])]) # determine the precipitation threshold mask if conditional: MASK_thr = np.logical_and.reduce( [R[i, :, :] >= R_thr for i in range(R.shape[0])]) else: MASK_thr = None # initialize the extrapolator x_values, y_values = np.meshgrid(np.arange(R.shape[2]), np.arange(R.shape[1])) xy_coords = np.stack([x_values, y_values]) extrap_kwargs = extrap_kwargs.copy() extrap_kwargs["xy_coords"] = xy_coords extrap_kwargs["allow_nonfinite_values"] = True # advect the previous precipitation fields to the same position with the # most recent one (i.e. transform them into the Lagrangian coordinates) res = list() def f(R, i): return extrapolator_method(R[i, :], V, ar_order - i, "min", **extrap_kwargs)[-1] for i in range(ar_order): if not DASK_IMPORTED: R[i, :, :] = f(R, i) else: res.append(dask.delayed(f)(R, i)) if DASK_IMPORTED: num_workers_ = len(res) if num_workers > len(res) else num_workers R = np.stack( list(dask.compute(*res, num_workers=num_workers_)) + [R[-1, :, :]]) # replace non-finite values with the minimum value R = R.copy() for i in range(R.shape[0]): R[i, ~np.isfinite(R[i, :])] = np.nanmin(R[i, :]) # compute the cascade decompositions of the input precipitation fields R_d = [] for i in range(ar_order + 1): R_ = decomp_method( R[i, :, :], filter, mask=MASK_thr, fft_method=fft, output_domain=domain, normalize=True, compute_stats=True, compact_output=True, ) R_d.append(R_) # rearrange the cascade levels into a four-dimensional array of shape # (n_cascade_levels,ar_order+1,m,n) for the autoregressive model R_c = nowcast_utils.stack_cascades(R_d, n_cascade_levels, convert_to_full_arrays=True) # compute lag-l temporal autocorrelation coefficients for each cascade level GAMMA = np.empty((n_cascade_levels, ar_order)) for i in range(n_cascade_levels): if domain == "spatial": GAMMA[i, :] = correlation.temporal_autocorrelation(R_c[i], mask=MASK_thr) else: GAMMA[i, :] = correlation.temporal_autocorrelation( R_c[i], domain="spectral", x_shape=R.shape[1:]) R_c = nowcast_utils.stack_cascades(R_d, n_cascade_levels, convert_to_full_arrays=False) R_d = R_d[-1] nowcast_utils.print_corrcoefs(GAMMA) if ar_order == 2: # adjust the lag-2 correlation coefficient to ensure that the AR(p) # process is stationary for i in range(n_cascade_levels): GAMMA[i, 1] = autoregression.adjust_lag2_corrcoef2( GAMMA[i, 0], GAMMA[i, 1]) # estimate the parameters of the AR(p) model from the autocorrelation # coefficients PHI = np.empty((n_cascade_levels, ar_order + 1)) for i in range(n_cascade_levels): PHI[i, :] = autoregression.estimate_ar_params_yw(GAMMA[i, :]) nowcast_utils.print_ar_params(PHI) # discard all except the p-1 last cascades because they are not needed for # the AR(p) model R_c = [R_c[i][-ar_order:] for i in range(n_cascade_levels)] D = None if probmatching_method == "mean": mu_0 = np.mean(R[-1, :, :][R[-1, :, :] >= R_thr]) # compute precipitation mask and wet area ratio MASK_p = R[-1, :, :] >= R_thr war = 1.0 * np.sum(MASK_p) / (R.shape[1] * R.shape[2]) if measure_time: init_time = time.time() - starttime_init R = R[-1, :, :] print("Starting nowcast computation.") if measure_time: starttime_mainloop = time.time() R_f = [] # iterate each time step for t in range(n_timesteps): print("Computing nowcast for time step %d... " % (t + 1), end="") sys.stdout.flush() if measure_time: starttime = time.time() for i in range(n_cascade_levels): R_c[i] = autoregression.iterate_ar_model(R_c[i], PHI[i, :]) R_d["cascade_levels"] = [ R_c[i][-1, :] for i in range(n_cascade_levels) ] if domain == "spatial": R_d["cascade_levels"] = np.stack(R_d["cascade_levels"]) R_c_ = recomp_method(R_d) if domain == "spectral": R_c_ = fft.irfft2(R_c_) MASK = _compute_sprog_mask(R_c_, war) R_c_[~MASK] = R_min if probmatching_method == "cdf": # adjust the CDF of the forecast to match the most recently # observed precipitation field R_c_ = probmatching.nonparam_match_empirical_cdf(R_c_, R) elif probmatching_method == "mean": mu_fct = np.mean(R_c_[MASK]) R_c_[MASK] = R_c_[MASK] - mu_fct + mu_0 R_c_[domain_mask] = np.nan # advect the recomposed precipitation field to obtain the forecast for # time step t extrap_kwargs.update({ "displacement_prev": D, "return_displacement": True }) R_f_, D_ = extrapolator_method(R_c_, V, 1, **extrap_kwargs) D = D_ R_f_ = R_f_[0] R_f.append(R_f_) if measure_time: print("%.2f seconds." % (time.time() - starttime)) else: print("done.") if measure_time: mainloop_time = time.time() - starttime_mainloop R_f = np.stack(R_f) if measure_time: return R_f, init_time, mainloop_time else: return R_f
def forecast( R, V, timesteps, n_ens_members=24, n_cascade_levels=6, R_thr=None, kmperpixel=None, timestep=None, extrap_method="semilagrangian", decomp_method="fft", bandpass_filter_method="gaussian", noise_method="nonparametric", noise_stddev_adj=None, ar_order=2, vel_pert_method="bps", conditional=False, probmatching_method="cdf", mask_method="incremental", callback=None, return_output=True, seed=None, num_workers=1, fft_method="numpy", domain="spatial", extrap_kwargs=None, filter_kwargs=None, noise_kwargs=None, vel_pert_kwargs=None, mask_kwargs=None, measure_time=False, ): """Generate a nowcast ensemble by using the Short-Term Ensemble Prediction System (STEPS) method. Parameters ---------- R: array-like Array of shape (ar_order+1,m,n) containing the input precipitation fields ordered by timestamp from oldest to newest. The time steps between the inputs are assumed to be regular. V: array-like Array of shape (2,m,n) containing the x- and y-components of the advection field. The velocities are assumed to represent one time step between the inputs. All values are required to be finite. timesteps: int or list of floats Number of time steps to forecast or a list of time steps for which the forecasts are computed (relative to the input time step). The elements of the list are required to be in ascending order. n_ens_members: int, optional The number of ensemble members to generate. n_cascade_levels: int, optional The number of cascade levels to use. R_thr: float, optional Specifies the threshold value for minimum observable precipitation intensity. Required if mask_method is not None or conditional is True. kmperpixel: float, optional Spatial resolution of the input data (kilometers/pixel). Required if vel_pert_method is not None or mask_method is 'incremental'. timestep: float, optional Time step of the motion vectors (minutes). Required if vel_pert_method is not None or mask_method is 'incremental'. extrap_method: str, optional Name of the extrapolation method to use. See the documentation of pysteps.extrapolation.interface. decomp_method: {'fft'}, optional Name of the cascade decomposition method to use. See the documentation of pysteps.cascade.interface. bandpass_filter_method: {'gaussian', 'uniform'}, optional Name of the bandpass filter method to use with the cascade decomposition. See the documentation of pysteps.cascade.interface. noise_method: {'parametric','nonparametric','ssft','nested',None}, optional Name of the noise generator to use for perturbating the precipitation field. See the documentation of pysteps.noise.interface. If set to None, no noise is generated. noise_stddev_adj: {'auto','fixed',None}, optional Optional adjustment for the standard deviations of the noise fields added to each cascade level. This is done to compensate incorrect std. dev. estimates of casace levels due to presence of no-rain areas. 'auto'=use the method implemented in pysteps.noise.utils.compute_noise_stddev_adjs. 'fixed'= use the formula given in :cite:`BPS2006` (eq. 6), None=disable noise std. dev adjustment. ar_order: int, optional The order of the autoregressive model to use. Must be >= 1. vel_pert_method: {'bps',None}, optional Name of the noise generator to use for perturbing the advection field. See the documentation of pysteps.noise.interface. If set to None, the advection field is not perturbed. conditional: bool, optional If set to True, compute the statistics of the precipitation field conditionally by excluding pixels where the values are below the threshold R_thr. mask_method: {'obs','sprog','incremental',None}, optional The method to use for masking no precipitation areas in the forecast field. The masked pixels are set to the minimum value of the observations. 'obs' = apply R_thr to the most recently observed precipitation intensity field, 'sprog' = use the smoothed forecast field from S-PROG, where the AR(p) model has been applied, 'incremental' = iteratively buffer the mask with a certain rate (currently it is 1 km/min), None=no masking. probmatching_method: {'cdf','mean',None}, optional Method for matching the statistics of the forecast field with those of the most recently observed one. 'cdf'=map the forecast CDF to the observed one, 'mean'=adjust only the conditional mean value of the forecast field in precipitation areas, None=no matching applied. Using 'mean' requires that mask_method is not None. callback: function, optional Optional function that is called after computation of each time step of the nowcast. The function takes one argument: a three-dimensional array of shape (n_ens_members,h,w), where h and w are the height and width of the input field R, respectively. This can be used, for instance, writing the outputs into files. return_output: bool, optional Set to False to disable returning the outputs as numpy arrays. This can save memory if the intermediate results are written to output files using the callback function. seed: int, optional Optional seed number for the random generators. num_workers: int, optional The number of workers to use for parallel computation. Applicable if dask is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it is advisable to disable OpenMP by setting the environment variable OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous threads. fft_method: str, optional A string defining the FFT method to use (see utils.fft.get_method). Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed, the recommended method is 'pyfftw'. domain: {"spatial", "spectral"} If "spatial", all computations are done in the spatial domain (the classical STEPS model). If "spectral", the AR(2) models and stochastic perturbations are applied directly in the spectral domain to reduce memory footprint and improve performance :cite:`PCH2019b`. extrap_kwargs: dict, optional Optional dictionary containing keyword arguments for the extrapolation method. See the documentation of pysteps.extrapolation. filter_kwargs: dict, optional Optional dictionary containing keyword arguments for the filter method. See the documentation of pysteps.cascade.bandpass_filters.py. noise_kwargs: dict, optional Optional dictionary containing keyword arguments for the initializer of the noise generator. See the documentation of pysteps.noise.fftgenerators. vel_pert_kwargs: dict, optional Optional dictionary containing keyword arguments 'p_par' and 'p_perp' for the initializer of the velocity perturbator. The choice of the optimal parameters depends on the domain and the used optical flow method. Default parameters from :cite:`BPS2006`: p_par = [10.88, 0.23, -7.68] p_perp = [5.76, 0.31, -2.72] Parameters fitted to the data (optical flow/domain): darts/fmi: p_par = [13.71259667, 0.15658963, -16.24368207] p_perp = [8.26550355, 0.17820458, -9.54107834] darts/mch: p_par = [24.27562298, 0.11297186, -27.30087471] p_perp = [-7.80797846e+01, -3.38641048e-02, 7.56715304e+01] darts/fmi+mch: p_par = [16.55447057, 0.14160448, -19.24613059] p_perp = [14.75343395, 0.11785398, -16.26151612] lucaskanade/fmi: p_par = [2.20837526, 0.33887032, -2.48995355] p_perp = [2.21722634, 0.32359621, -2.57402761] lucaskanade/mch: p_par = [2.56338484, 0.3330941, -2.99714349] p_perp = [1.31204508, 0.3578426, -1.02499891] lucaskanade/fmi+mch: p_par = [2.31970635, 0.33734287, -2.64972861] p_perp = [1.90769947, 0.33446594, -2.06603662] vet/fmi: p_par = [0.25337388, 0.67542291, 11.04895538] p_perp = [0.02432118, 0.99613295, 7.40146505] vet/mch: p_par = [0.5075159, 0.53895212, 7.90331791] p_perp = [0.68025501, 0.41761289, 4.73793581] vet/fmi+mch: p_par = [0.29495222, 0.62429207, 8.6804131 ] p_perp = [0.23127377, 0.59010281, 5.98180004] fmi=Finland, mch=Switzerland, fmi+mch=both pooled into the same data set The above parameters have been fitten by using run_vel_pert_analysis.py and fit_vel_pert_params.py located in the scripts directory. See pysteps.noise.motion for additional documentation. mask_kwargs: dict Optional dictionary containing mask keyword arguments 'mask_f' and 'mask_rim', the factor defining the the mask increment and the rim size, respectively. The mask increment is defined as mask_f*timestep/kmperpixel. measure_time: bool If set to True, measure, print and return the computation time. Returns ------- out: ndarray If return_output is True, a four-dimensional array of shape (n_ens_members,num_timesteps,m,n) containing a time series of forecast precipitation fields for each ensemble member. Otherwise, a None value is returned. The time series starts from t0+timestep, where timestep is taken from the input precipitation fields R. If measure_time is True, the return value is a three-element tuple containing the nowcast array, the initialization time of the nowcast generator and the time used in the main loop (seconds). See also -------- pysteps.extrapolation.interface, pysteps.cascade.interface, pysteps.noise.interface, pysteps.noise.utils.compute_noise_stddev_adjs References ---------- :cite:`Seed2003`, :cite:`BPS2006`, :cite:`SPN2013`, :cite:`PCH2019b` """ _check_inputs(R, V, timesteps, ar_order) if extrap_kwargs is None: extrap_kwargs = dict() if filter_kwargs is None: filter_kwargs = dict() if noise_kwargs is None: noise_kwargs = dict() if vel_pert_kwargs is None: vel_pert_kwargs = dict() if mask_kwargs is None: mask_kwargs = dict() if np.any(~np.isfinite(V)): raise ValueError("V contains non-finite values") if mask_method not in ["obs", "sprog", "incremental", None]: raise ValueError( "unknown mask method %s: must be 'obs', 'sprog' or 'incremental' or None" % mask_method ) if conditional and R_thr is None: raise ValueError("conditional=True but R_thr is not set") if mask_method is not None and R_thr is None: raise ValueError("mask_method!=None but R_thr=None") if noise_stddev_adj not in ["auto", "fixed", None]: raise ValueError( "unknown noise_std_dev_adj method %s: must be 'auto', 'fixed', or None" % noise_stddev_adj ) if kmperpixel is None: if vel_pert_method is not None: raise ValueError("vel_pert_method is set but kmperpixel=None") if mask_method == "incremental": raise ValueError("mask_method='incremental' but kmperpixel=None") if timestep is None: if vel_pert_method is not None: raise ValueError("vel_pert_method is set but timestep=None") if mask_method == "incremental": raise ValueError("mask_method='incremental' but timestep=None") print("Computing STEPS nowcast:") print("------------------------") print("") print("Inputs:") print("-------") print("input dimensions: %dx%d" % (R.shape[1], R.shape[2])) if kmperpixel is not None: print("km/pixel: %g" % kmperpixel) if timestep is not None: print("time step: %d minutes" % timestep) print("") print("Methods:") print("--------") print("extrapolation: %s" % extrap_method) print("bandpass filter: %s" % bandpass_filter_method) print("decomposition: %s" % decomp_method) print("noise generator: %s" % noise_method) print("noise adjustment: %s" % ("yes" if noise_stddev_adj else "no")) print("velocity perturbator: %s" % vel_pert_method) print("conditional statistics: %s" % ("yes" if conditional else "no")) print("precip. mask method: %s" % mask_method) print("probability matching: %s" % probmatching_method) print("FFT method: %s" % fft_method) print("domain: %s" % domain) print("") print("Parameters:") print("-----------") if isinstance(timesteps, int): print("number of time steps: %d" % timesteps) else: print("time steps: %s" % timesteps) print("ensemble size: %d" % n_ens_members) print("parallel threads: %d" % num_workers) print("number of cascade levels: %d" % n_cascade_levels) print("order of the AR(p) model: %d" % ar_order) if vel_pert_method == "bps": vp_par = vel_pert_kwargs.get("p_par", noise.motion.get_default_params_bps_par()) vp_perp = vel_pert_kwargs.get( "p_perp", noise.motion.get_default_params_bps_perp() ) print( "velocity perturbations, parallel: %g,%g,%g" % (vp_par[0], vp_par[1], vp_par[2]) ) print( "velocity perturbations, perpendicular: %g,%g,%g" % (vp_perp[0], vp_perp[1], vp_perp[2]) ) if conditional or mask_method is not None: print("precip. intensity threshold: %g" % R_thr) num_ensemble_workers = n_ens_members if num_workers > n_ens_members else num_workers if measure_time: starttime_init = time.time() fft = utils.get_method(fft_method, shape=R.shape[1:], n_threads=num_workers) M, N = R.shape[1:] # initialize the band-pass filter filter_method = cascade.get_method(bandpass_filter_method) filter = filter_method((M, N), n_cascade_levels, **filter_kwargs) decomp_method, recomp_method = cascade.get_method(decomp_method) extrapolator_method = extrapolation.get_method(extrap_method) x_values, y_values = np.meshgrid(np.arange(R.shape[2]), np.arange(R.shape[1])) xy_coords = np.stack([x_values, y_values]) R = R[-(ar_order + 1) :, :, :].copy() # determine the domain mask from non-finite values domain_mask = np.logical_or.reduce( [~np.isfinite(R[i, :]) for i in range(R.shape[0])] ) # determine the precipitation threshold mask if conditional: MASK_thr = np.logical_and.reduce( [R[i, :, :] >= R_thr for i in range(R.shape[0])] ) else: MASK_thr = None # advect the previous precipitation fields to the same position with the # most recent one (i.e. transform them into the Lagrangian coordinates) extrap_kwargs = extrap_kwargs.copy() extrap_kwargs["xy_coords"] = xy_coords extrap_kwargs["allow_nonfinite_values"] = True res = list() def f(R, i): return extrapolator_method(R[i, :, :], V, ar_order - i, "min", **extrap_kwargs)[ -1 ] for i in range(ar_order): if not DASK_IMPORTED: R[i, :, :] = f(R, i) else: res.append(dask.delayed(f)(R, i)) if DASK_IMPORTED: num_workers_ = len(res) if num_workers > len(res) else num_workers R = np.stack(list(dask.compute(*res, num_workers=num_workers_)) + [R[-1, :, :]]) # replace non-finite values with the minimum value R = R.copy() for i in range(R.shape[0]): R[i, ~np.isfinite(R[i, :])] = np.nanmin(R[i, :]) if noise_method is not None: # get methods for perturbations init_noise, generate_noise = noise.get_method(noise_method) # initialize the perturbation generator for the precipitation field pp = init_noise(R, fft_method=fft, **noise_kwargs) if noise_stddev_adj == "auto": print("Computing noise adjustment coefficients... ", end="", flush=True) if measure_time: starttime = time.time() R_min = np.min(R) noise_std_coeffs = noise.utils.compute_noise_stddev_adjs( R[-1, :, :], R_thr, R_min, filter, decomp_method, pp, generate_noise, 20, conditional=True, num_workers=num_workers, ) if measure_time: print("%.2f seconds." % (time.time() - starttime)) else: print("done.") elif noise_stddev_adj == "fixed": f = lambda k: 1.0 / (0.75 + 0.09 * k) noise_std_coeffs = [f(k) for k in range(1, n_cascade_levels + 1)] else: noise_std_coeffs = np.ones(n_cascade_levels) if noise_stddev_adj is not None: print("noise std. dev. coeffs: %s" % str(noise_std_coeffs)) # compute the cascade decompositions of the input precipitation fields R_d = [] for i in range(ar_order + 1): R_ = decomp_method( R[i, :, :], filter, mask=MASK_thr, fft_method=fft, output_domain=domain, normalize=True, compute_stats=True, compact_output=True, ) R_d.append(R_) # normalize the cascades and rearrange them into a four-dimensional array # of shape (n_cascade_levels,ar_order+1,m,n) for the autoregressive model R_c = nowcast_utils.stack_cascades(R_d, n_cascade_levels) R_d = R_d[-1] R_d = [R_d.copy() for j in range(n_ens_members)] # compute lag-l temporal autocorrelation coefficients for each cascade level GAMMA = np.empty((n_cascade_levels, ar_order)) for i in range(n_cascade_levels): GAMMA[i, :] = correlation.temporal_autocorrelation(R_c[i], mask=MASK_thr) nowcast_utils.print_corrcoefs(GAMMA) if ar_order == 2: # adjust the lag-2 correlation coefficient to ensure that the AR(p) # process is stationary for i in range(n_cascade_levels): GAMMA[i, 1] = autoregression.adjust_lag2_corrcoef2(GAMMA[i, 0], GAMMA[i, 1]) # estimate the parameters of the AR(p) model from the autocorrelation # coefficients PHI = np.empty((n_cascade_levels, ar_order + 1)) for i in range(n_cascade_levels): PHI[i, :] = autoregression.estimate_ar_params_yw(GAMMA[i, :]) nowcast_utils.print_ar_params(PHI) # discard all except the p-1 last cascades because they are not needed for # the AR(p) model R_c = [R_c[i][-ar_order:] for i in range(n_cascade_levels)] # stack the cascades into a list containing all ensemble members R_c = [ [R_c[j].copy() for j in range(n_cascade_levels)] for i in range(n_ens_members) ] # initialize the random generators if noise_method is not None: randgen_prec = [] randgen_motion = [] np.random.seed(seed) for j in range(n_ens_members): rs = np.random.RandomState(seed) randgen_prec.append(rs) seed = rs.randint(0, high=1e9) rs = np.random.RandomState(seed) randgen_motion.append(rs) seed = rs.randint(0, high=1e9) if vel_pert_method is not None: init_vel_noise, generate_vel_noise = noise.get_method(vel_pert_method) # initialize the perturbation generators for the motion field vps = [] for j in range(n_ens_members): kwargs = { "randstate": randgen_motion[j], "p_par": vp_par, "p_perp": vp_perp, } vp_ = init_vel_noise(V, 1.0 / kmperpixel, timestep, **kwargs) vps.append(vp_) D = [None for j in range(n_ens_members)] R_f = [[] for j in range(n_ens_members)] if probmatching_method == "mean": mu_0 = np.mean(R[-1, :, :][R[-1, :, :] >= R_thr]) R_m = None if mask_method is not None: MASK_prec = R[-1, :, :] >= R_thr if mask_method == "obs": pass elif mask_method == "sprog": # compute the wet area ratio and the precipitation mask war = 1.0 * np.sum(MASK_prec) / (R.shape[1] * R.shape[2]) R_m = [R_c[0][i].copy() for i in range(n_cascade_levels)] R_m_d = R_d[0].copy() elif mask_method == "incremental": # get mask parameters mask_rim = mask_kwargs.get("mask_rim", 10) mask_f = mask_kwargs.get("mask_f", 1.0) # initialize the structuring element struct = scipy.ndimage.generate_binary_structure(2, 1) # iterate it to expand it nxn n = mask_f * timestep / kmperpixel struct = scipy.ndimage.iterate_structure(struct, int((n - 1) / 2.0)) # initialize precip mask for each member MASK_prec = _compute_incremental_mask(MASK_prec, struct, mask_rim) MASK_prec = [MASK_prec.copy() for j in range(n_ens_members)] if noise_method is None and R_m is None: R_m = [R_c[0][i].copy() for i in range(n_cascade_levels)] fft_objs = [] for i in range(n_ens_members): fft_objs.append(utils.get_method(fft_method, shape=R.shape[1:])) if measure_time: init_time = time.time() - starttime_init R = R[-1, :, :] print("Starting nowcast computation.") if measure_time: starttime_mainloop = time.time() if isinstance(timesteps, int): timesteps = range(timesteps + 1) timestep_type = "int" else: original_timesteps = [0] + list(timesteps) timesteps = nowcast_utils.binned_timesteps(original_timesteps) timestep_type = "list" extrap_kwargs["return_displacement"] = True R_f_prev = [R for i in range(n_ens_members)] t_prev = [0.0 for j in range(n_ens_members)] t_total = [0.0 for j in range(n_ens_members)] # iterate each time step for t, subtimestep_idx in enumerate(timesteps): if timestep_type == "list": subtimesteps = [original_timesteps[t_] for t_ in subtimestep_idx] else: subtimesteps = [t] if (timestep_type == "list" and subtimesteps) or ( timestep_type == "int" and t > 0 ): is_nowcast_time_step = True else: is_nowcast_time_step = False if is_nowcast_time_step: print( "Computing nowcast for time step %d... " % t, end="", flush=True, ) if measure_time: starttime = time.time() if noise_method is None or mask_method == "sprog": for i in range(n_cascade_levels): # use a separate AR(p) model for the non-perturbed forecast, # from which the mask is obtained R_m[i] = autoregression.iterate_ar_model(R_m[i], PHI[i, :]) R_m_d["cascade_levels"] = [R_m[i][-1] for i in range(n_cascade_levels)] if domain == "spatial": R_m_d["cascade_levels"] = np.stack(R_m_d["cascade_levels"]) R_m_ = recomp_method(R_m_d) if domain == "spectral": R_m_ = fft.irfft2(R_m_) if mask_method == "sprog": MASK_prec = _compute_sprog_mask(R_m_, war) # the nowcast iteration for each ensemble member def worker(j): if noise_method is not None: # generate noise field EPS = generate_noise( pp, randstate=randgen_prec[j], fft_method=fft_objs[j], domain=domain ) # decompose the noise field into a cascade EPS = decomp_method( EPS, filter, fft_method=fft_objs[j], input_domain=domain, output_domain=domain, compute_stats=True, normalize=True, compact_output=True, ) else: EPS = None # iterate the AR(p) model for each cascade level for i in range(n_cascade_levels): # normalize the noise cascade if EPS is not None: EPS_ = EPS["cascade_levels"][i] EPS_ *= noise_std_coeffs[i] else: EPS_ = None # apply AR(p) process to cascade level if EPS is not None or vel_pert_method is not None: R_c[j][i] = autoregression.iterate_ar_model( R_c[j][i], PHI[i, :], eps=EPS_ ) else: # use the deterministic AR(p) model computed above if # perturbations are disabled R_c[j][i] = R_m[i] EPS = None EPS_ = None # compute the recomposed precipitation field(s) from the cascades # obtained from the AR(p) model(s) R_d[j]["cascade_levels"] = [ R_c[j][i][-1, :] for i in range(n_cascade_levels) ] if domain == "spatial": R_d[j]["cascade_levels"] = np.stack(R_d[j]["cascade_levels"]) R_f_new = recomp_method(R_d[j]) if domain == "spectral": R_f_new = fft_objs[j].irfft2(R_f_new) if mask_method is not None: # apply the precipitation mask to prevent generation of new # precipitation into areas where it was not originally # observed R_cmin = R_f_new.min() if mask_method == "incremental": R_f_new = R_cmin + (R_f_new - R_cmin) * MASK_prec[j] MASK_prec_ = R_f_new > R_cmin else: MASK_prec_ = MASK_prec # Set to min value outside of mask R_f_new[~MASK_prec_] = R_cmin if probmatching_method == "cdf": # adjust the CDF of the forecast to match the most recently # observed precipitation field R_f_new = probmatching.nonparam_match_empirical_cdf(R_f_new, R) elif probmatching_method == "mean": MASK = R_f_new >= R_thr mu_fct = np.mean(R_f_new[MASK]) R_f_new[MASK] = R_f_new[MASK] - mu_fct + mu_0 if mask_method == "incremental": MASK_prec[j] = _compute_incremental_mask( R_f_new >= R_thr, struct, mask_rim ) R_f_new[domain_mask] = np.nan R_f_out = [] extrap_kwargs_ = extrap_kwargs.copy() V_pert = V # advect the recomposed precipitation field to obtain the forecast for # the current time step (or subtimesteps if non-integer time steps are # given) for t_sub in subtimesteps: if t_sub > 0: t_diff_prev_int = t_sub - int(t_sub) if t_diff_prev_int > 0.0: R_f_ip = (1.0 - t_diff_prev_int) * R_f_prev[ j ] + t_diff_prev_int * R_f_new else: R_f_ip = R_f_prev[j] t_diff_prev = t_sub - t_prev[j] t_total[j] += t_diff_prev # compute the perturbed motion field if vel_pert_method is not None: V_pert = V + generate_vel_noise(vps[j], t_total[j] * timestep) extrap_kwargs_["displacement_prev"] = D[j] R_f_ep, D[j] = extrapolator_method( R_f_ip, V_pert, [t_diff_prev], **extrap_kwargs_, ) R_f_out.append(R_f_ep[0]) t_prev[j] = t_sub # advect the forecast field by one time step if no subtimesteps in the # current interval were found if not subtimesteps: t_diff_prev = t + 1 - t_prev[j] t_total[j] += t_diff_prev # compute the perturbed motion field if vel_pert_method is not None: V_pert = V + generate_vel_noise(vps[j], t_total[j] * timestep) extrap_kwargs_["displacement_prev"] = D[j] _, D[j] = extrapolator_method( None, V_pert, [t_diff_prev], **extrap_kwargs_, ) t_prev[j] = t + 1 R_f_prev[j] = R_f_new return R_f_out res = [] for j in range(n_ens_members): if not DASK_IMPORTED or n_ens_members == 1: res.append(worker(j)) else: res.append(dask.delayed(worker)(j)) R_f_ = ( dask.compute(*res, num_workers=num_ensemble_workers) if DASK_IMPORTED and n_ens_members > 1 else res ) res = None if is_nowcast_time_step: if measure_time: print("%.2f seconds." % (time.time() - starttime)) else: print("done.") if callback is not None: R_f_stacked = np.stack(R_f_) if R_f_stacked.shape[1] > 0: callback(R_f_stacked.squeeze()) if return_output: for j in range(n_ens_members): R_f[j].extend(R_f_[j]) R_f_ = None if measure_time: mainloop_time = time.time() - starttime_mainloop if return_output: outarr = np.stack([np.stack(R_f[j]) for j in range(n_ens_members)]) if measure_time: return outarr, init_time, mainloop_time else: return outarr else: return None
def decomposition_fft(field, bp_filter, **kwargs): """Decompose a 2d input field into multiple spatial scales by using the Fast Fourier Transform (FFT) and a bandpass filter. Parameters ---------- field : array_like Two-dimensional array containing the input field. All values are required to be finite. bp_filter : dict A filter returned by a method implemented in :py:mod:`pysteps.cascade.bandpass_filters`. Other Parameters ---------------- fft_method : str or tuple A string or a (function,kwargs) tuple defining the FFT method to use (see :py:func:`pysteps.utils.interface.get_method`). Defaults to "numpy". MASK : array_like Optional mask to use for computing the statistics for the cascade levels. Pixels with MASK==False are excluded from the computations. Returns ------- out : ndarray A dictionary described in the module documentation. The number of cascade levels is determined from the filter (see :py:mod:`pysteps.cascade.bandpass_filters`). """ fft = kwargs.get("fft_method", "numpy") if type(fft) == str: fft = utils.get_method(fft, shape=field.shape) mask = kwargs.get("MASK", None) if len(field.shape) != 2: raise ValueError("The input is not two-dimensional array") if mask is not None and mask.shape != field.shape: raise ValueError("Dimension mismatch between X and MASK:" + "X.shape=" + str(field.shape) + ",mask.shape" + str(mask.shape)) if field.shape[0] != bp_filter["weights_2d"].shape[1]: raise ValueError( "dimension mismatch between X and filter: " + "X.shape[0]=%d , " % field.shape[0] + "filter['weights_2d'].shape[1]" "=%d" % bp_filter["weights_2d"].shape[1]) if int(field.shape[1] / 2) + 1 != bp_filter["weights_2d"].shape[2]: raise ValueError( "Dimension mismatch between X and filter: " "int(X.shape[1]/2)+1=%d , " % (int(field.shape[1] / 2) + 1) + "filter['weights_2d'].shape[2]" "=%d" % bp_filter["weights_2d"].shape[2]) if np.any(~np.isfinite(field)): raise ValueError("X contains non-finite values") result = {} means = [] stds = [] field_decomp = [] for k in range(len(bp_filter["weights_1d"])): _decomp_field = fft.irfft2(fft.rfft2(field) * bp_filter["weights_2d"][k, :, :]) field_decomp.append(_decomp_field) if mask is not None: _decomp_field = _decomp_field[mask] means.append(np.mean(_decomp_field)) stds.append(np.std(_decomp_field)) result["cascade_levels"] = np.stack(field_decomp) result["means"] = means result["stds"] = stds return result
def DARTS(input_images, **kwargs): """Compute the advection field from a sequence of input images by using the DARTS method. :cite:`RCW2011` Parameters ---------- input_images: array-like Array of shape (T,m,n) containing a sequence of T two-dimensional input images of shape (m,n). Other Parameters ---------------- N_x: int Number of DFT coefficients to use for the input images, x-axis (default=50). N_y: int Number of DFT coefficients to use for the input images, y-axis (default=50). N_t: int Number of DFT coefficients to use for the input images, time axis (default=4). N_t must be strictly smaller than T. M_x: int Number of DFT coefficients to compute for the output advection field, x-axis (default=2). M_y: int Number of DFT coefficients to compute for the output advection field, y-axis (default=2). fft_method: str A string defining the FFT method to use, see utils.fft.get_method. Defaults to 'numpy'. output_type: {"spatial", "spectral"} The type of the output: "spatial"=apply the inverse FFT to obtain the spatial representation of the advection field, "spectral"=return the (truncated) DFT representation. n_threads: int Number of threads to use for the FFT computation. Applicable if fft_method is 'pyfftw'. verbose: bool If True, print information messages. lsq_method: {1, 2} The method to use for solving the linear equations in the least squares sense: 1=numpy.linalg.lstsq, 2=explicit computation of the Moore-Penrose pseudoinverse and SVD. verbose: bool if set to True, it prints information about the program Returns ------- out: ndarray Three-dimensional array (2,m,n) containing the dense x- and y-components of the motion field in units of pixels / timestep as given by the input array R. """ N_x = kwargs.get("N_x", 50) N_y = kwargs.get("N_y", 50) N_t = kwargs.get("N_t", 4) M_x = kwargs.get("M_x", 2) M_y = kwargs.get("M_y", 2) fft_method = kwargs.get("fft_method", "numpy") output_type = kwargs.get("output_type", "spatial") lsq_method = kwargs.get("lsq_method", 2) verbose = kwargs.get("verbose", True) if N_t >= input_images.shape[0] - 1: raise ValueError("N_t = %d >= %d = T-1, but N_t < T-1 required" % (N_t, input_images.shape[0] - 1)) if output_type not in ["spatial", "spectral"]: raise ValueError( "invalid output_type=%s, must be 'spatial' or 'spectral'" % output_type) if verbose: print("Computing the motion field with the DARTS method.") t0 = time.time() input_images = np.moveaxis(input_images, (0, 1, 2), (2, 0, 1)) fft = utils.get_method( fft_method, shape=input_images.shape[:2], fftn_shape=input_images.shape, **kwargs, ) T_x = input_images.shape[1] T_y = input_images.shape[0] T_t = input_images.shape[2] if verbose: print("-----") print("DARTS") print("-----") print(" Computing the FFT of the reflectivity fields...", end="", flush=True) starttime = time.time() input_images = fft.fftn(input_images) if verbose: print("Done in %.2f seconds." % (time.time() - starttime)) print(" Constructing the y-vector...", end="", flush=True) starttime = time.time() m = (2 * N_x + 1) * (2 * N_y + 1) * (2 * N_t + 1) n = (2 * M_x + 1) * (2 * M_y + 1) y = np.zeros(m, dtype=complex) k_t, k_y, k_x = np.unravel_index(np.arange(m), (2 * N_t + 1, 2 * N_y + 1, 2 * N_x + 1)) for i in range(m): k_x_ = k_x[i] - N_x k_y_ = k_y[i] - N_y k_t_ = k_t[i] - N_t y[i] = k_t_ * input_images[k_y_, k_x_, k_t_] if verbose: print("Done in %.2f seconds." % (time.time() - starttime)) A = np.zeros((m, n), dtype=complex) B = np.zeros((m, n), dtype=complex) if verbose: print(" Constructing the H-matrix...", end="", flush=True) starttime = time.time() c1 = -1.0 * T_t / (T_x * T_y) kp_y, kp_x = np.unravel_index(np.arange(n), (2 * M_y + 1, 2 * M_x + 1)) for i in range(m): k_x_ = k_x[i] - N_x k_y_ = k_y[i] - N_y k_t_ = k_t[i] - N_t kp_x_ = kp_x[:] - M_x kp_y_ = kp_y[:] - M_y i_ = k_y_ - kp_y_ j_ = k_x_ - kp_x_ R_ = input_images[i_, j_, k_t_] c2 = c1 / T_y * i_ A[i, :] = c2 * R_ c2 = c1 / T_x * j_ B[i, :] = c2 * R_ if verbose: print("Done in %.2f seconds." % (time.time() - starttime)) print(" Solving the linear systems...", end="", flush=True) starttime = time.time() if lsq_method == 1: x = lstsq(np.hstack([A, B]), y, rcond=0.01)[0] else: x = _leastsq(A, B, y) if verbose: print("Done in %.2f seconds." % (time.time() - starttime)) h, w = 2 * M_y + 1, 2 * M_x + 1 U = np.zeros((h, w), dtype=complex) V = np.zeros((h, w), dtype=complex) i, j = np.unravel_index(np.arange(h * w), (h, w)) V[i, j] = x[0:h * w] U[i, j] = x[h * w:2 * h * w] k_x, k_y = np.meshgrid(np.arange(-M_x, M_x + 1), np.arange(-M_y, M_y + 1)) if output_type == "spatial": U = np.real( fft.ifft2( _fill(U, input_images.shape[0], input_images.shape[1], k_x, k_y))) V = np.real( fft.ifft2( _fill(V, input_images.shape[0], input_images.shape[1], k_x, k_y))) if verbose: print("--- %s seconds ---" % (time.time() - t0)) return np.stack([U, V])
obs_fns = (obs_fns[0][1:], obs_fns[1][1:]) if len(obs_fns[0]) == 0: curdate += timedelta(minutes=timestep) print("Skipping, no verifying observations found.") continue R_obs, _, metaobs = io.readers.read_timeseries( obs_fns, importer, **datasource["importer_kwargs"]) R_obs, metaobs = conversion.to_rainrate(R_obs, metaobs) R_obs[R_obs < R_min] = 0.0 metaobs["threshold"] = R_min # Compute observed accumulations metafct = metaobs.copy() if v_accu_min > time_res: aggregator = utils.get_method("accumulate") R_obs, metaobs = aggregator(R_obs, metaobs, v_accu_min) R_obs[R_obs < R_min] = 0.0 metaobs["threshold"] = R_min if R_obs is None: curdate += timedelta(minutes=timestep) print("Skipping, no verifying observations found.") continue for c in cascade_levels: print('----------------------------------') if c == 1: bandpass_filter = "uniform" else: