def test_decompose_recompose():
"""Tests cascade decomposition."""
pytest.importorskip("netCDF4")
root_path = pysteps.rcparams.data_sources["bom"]["root_path"]
rel_path = os.path.join("prcp-cscn", "2", "2018", "06", "16")
filename = os.path.join(root_path, rel_path, "2_20180616_120000.prcp-cscn.nc")
precip, _, metadata = pysteps.io.import_bom_rf3(filename)
# Convert to rain rate from mm
precip, metadata = pysteps.utils.to_rainrate(precip, metadata)
# Log-transform the data
precip, metadata = pysteps.utils.dB_transform(
precip, metadata, threshold=0.1, zerovalue=-15.0
)
# Set Nans as the fill value
precip[~np.isfinite(precip)] = metadata["zerovalue"]
# Set number of cascade levels
num_cascade_levels = 9
# Construct the Gaussian bandpass filters
_filter = filter_gaussian(precip.shape, num_cascade_levels)
# Decompose precip
decomp = decomposition_fft(precip, _filter)
# Recomposed precip from decomp
recomposed = recompose_fft(decomp)
# Assert
assert_array_almost_equal(recomposed.squeeze(), precip)
fn_ext = "pgm.gz"
cmap = cm.RdBu_r
vmin = -3
vmax = 3
fn = io.archive.find_by_date(date, root_path, "%Y%m%d", fn_pattern, fn_ext, 5)
R, _, metadata = io.read_timeseries(fn, import_fmi_pgm, gzipped=True)
R = R.squeeze()
R[R < 10.0] = 5.0
R[~np.isfinite(R)] = 5.0
R = (R - np.mean(R)) / np.std(R)
filter = filter_gaussian(R.shape, 8)
decomp = decomposition_fft(R, filter)
for i in range(8):
mu = decomp["means"][i]
sigma = decomp["stds"][i]
decomp["cascade_levels"][i] = (decomp["cascade_levels"][i] - mu) / sigma
fig, ax = pyplot.subplots(nrows=2, ncols=4)
im = ax[0, 0].imshow(R, cmap=cmap, vmin=vmin, vmax=vmax)
ax[0, 1].imshow(decomp["cascade_levels"][0],
cmap=cmap,
vmin=vmin,
vmax=vmax,
rasterized=True)
ax.set_xlabel("Wavenumber $k_x$")
ax.set_ylabel("Wavenumber $k_y$")
ax.set_title("Log-power spectrum of R")
plt.show()
###############################################################################
# Cascade decomposition
# ---------------------
#
# First, construct a set of Gaussian bandpass filters and plot the corresponding
# 1D filters.
num_cascade_levels = 7
# Construct the Gaussian bandpass filters
filter = filter_gaussian(R.shape, num_cascade_levels)
# Plot the bandpass filter weights
L = max(N, M)
fig, ax = plt.subplots()
for k in range(num_cascade_levels):
ax.semilogx(
np.linspace(0, L / 2, len(filter["weights_1d"][k, :])),
filter["weights_1d"][k, :],
"k-",
basex=pow(0.5 * L / 3, 1.0 / (num_cascade_levels - 2)),
)
ax.set_xlim(1, L / 2)
ax.set_ylim(0, 1)
xt = np.hstack([[1.0], filter["central_wavenumbers"][1:]])
ax.set_xticks(xt)
def decompose_NWP(
R_NWP,
NWP_model,
analysis_time,
timestep,
valid_times,
output_path,
num_cascade_levels=8,
num_workers=1,
decomp_method="fft",
fft_method="numpy",
domain="spatial",
normalize=True,
compute_stats=True,
compact_output=True,
):
"""Decomposes the NWP forecast data into cascades and saves it in
a netCDF file
Parameters
----------
R_NWP: array-like
Array of dimension (n_timesteps, x, y) containing the precipitation forecast
from some NWP model.
NWP_model: str
The name of the NWP model
analysis_time: numpy.datetime64
The analysis time of the NWP forecast. The analysis time is assumed to be a
numpy.datetime64 type as imported by the pysteps importer
timestep: int
Timestep in minutes between subsequent NWP forecast fields
valid_times: array_like
Array containing the valid times of the NWP forecast fields. The times are
assumed to be numpy.datetime64 types as imported by the pysteps importer.
output_path: str
The location where to save the file with the NWP cascade. Defaults to the
path_workdir specified in the rcparams file.
num_cascade_levels: int, optional
The number of frequency bands to use. Must be greater than 2. Defaults to 8.
num_workers: int, optional
The number of workers to use for parallel computation. Applicable if dask
is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it
is advisable to disable OpenMP by setting the environment variable
OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous
threads.
Other Parameters
----------------
decomp_method: str, optional
A string defining the decomposition method to use. Defaults to "fft".
fft_method: str or tuple, optional
A string or a (function,kwargs) tuple defining the FFT method to use
(see :py:func:`pysteps.utils.interface.get_method`).
Defaults to "numpy". This option is not used if input_domain and
output_domain are both set to "spectral".
domain: {"spatial", "spectral"}, optional
If "spatial", the output cascade levels are transformed back to the
spatial domain by using the inverse FFT. If "spectral", the cascade is
kept in the spectral domain. Defaults to "spatial".
normalize: bool, optional
If True, normalize the cascade levels to zero mean and unit variance.
Requires that compute_stats is True. Implies that compute_stats is True.
Defaults to False.
compute_stats: bool, optional
If True, the output dictionary contains the keys "means" and "stds"
for the mean and standard deviation of each output cascade level.
Defaults to False.
compact_output: bool, optional
Applicable if output_domain is "spectral". If set to True, only the
parts of the Fourier spectrum with non-negligible filter weights are
stored. Defaults to False.
Returns
-------
None
"""
if not NETCDF4_IMPORTED:
raise MissingOptionalDependency(
"netCDF4 package is required to save the decomposed NWP data, "
"but it is not installed")
# Make a NetCDF file
output_date = f"{analysis_time.astype('datetime64[us]').astype(datetime.datetime):%Y%m%d%H%M%S}"
outfn = Path(output_path) / f"cascade_{NWP_model}_{output_date}.nc"
ncf = netCDF4.Dataset(outfn, "w", format="NETCDF4")
# Express times relative to the zero time
zero_time = np.datetime64("1970-01-01T00:00:00", "ns")
valid_times = np.array(valid_times) - zero_time
analysis_time = analysis_time - zero_time
# Set attributes of decomposition method
ncf.domain = domain
ncf.normalized = int(normalize)
ncf.compact_output = int(compact_output)
ncf.analysis_time = int(analysis_time)
ncf.timestep = int(timestep)
# Create dimensions
ncf.createDimension("time", R_NWP.shape[0])
ncf.createDimension("cascade_levels", num_cascade_levels)
ncf.createDimension("x", R_NWP.shape[2])
ncf.createDimension("y", R_NWP.shape[1])
# Create variables (decomposed cascade, means and standard deviations)
R_d = ncf.createVariable(
"pr_decomposed",
np.float32,
("time", "cascade_levels", "y", "x"),
zlib=True,
complevel=4,
)
means = ncf.createVariable("means", np.float64, ("time", "cascade_levels"))
stds = ncf.createVariable("stds", np.float64, ("time", "cascade_levels"))
v_times = ncf.createVariable("valid_times", np.float64, ("time", ))
v_times.units = "nanoseconds since 1970-01-01 00:00:00"
# The valid times are saved as an array of floats, because netCDF files can't handle datetime types
v_times[:] = np.array(
[np.float64(valid_times[i]) for i in range(len(valid_times))])
# Decompose the NWP data
filter_g = filter_gaussian(R_NWP.shape[1:], num_cascade_levels)
fft = utils_get_method(fft_method,
shape=R_NWP.shape[1:],
n_threads=num_workers)
decomp_method, _ = cascade_get_method(decomp_method)
for i in range(R_NWP.shape[0]):
R_ = decomp_method(
field=R_NWP[i, :, :],
bp_filter=filter_g,
fft_method=fft,
input_domain=domain,
output_domain=domain,
normalize=normalize,
compute_stats=compute_stats,
compact_output=compact_output,
)
# Save data to netCDF file
# print(R_["cascade_levels"])
R_d[i, :, :, :] = R_["cascade_levels"]
means[i, :] = R_["means"]
stds[i, :] = R_["stds"]
# Close the file
ncf.close()
fns = io.archive.find_by_date(date,
root_path,
"%Y%m%d",
fn_pattern,
fn_ext,
5,
num_prev_files=9)
# read the radar composites and apply thresholding
Z, _, metadata = io.read_timeseries(fns, import_fmi_pgm, gzipped=True)
R = conversion.to_rainrate(Z, metadata, 223.0, 1.53)[0]
R = transformation.dB_transform(R, threshold=0.1, zerovalue=-15.0)[0]
R[~np.isfinite(R)] = -15.0
# construct bandpass filter and apply the cascade decomposition
filter = filter_gaussian(R.shape[1:], 7)
decomp = decomposition_fft(R[-1, :, :], filter)
# plot the normalized cascade levels
for i in range(7):
mu = decomp["means"][i]
sigma = decomp["stds"][i]
decomp["cascade_levels"][i] = (decomp["cascade_levels"][i] - mu) / sigma
fig, ax = pyplot.subplots(nrows=2, ncols=4)
ax[0, 0].imshow(R[-1, :, :], cmap=cm.RdBu_r, vmin=-3, vmax=3)
ax[0, 1].imshow(decomp["cascade_levels"][0], cmap=cm.RdBu_r, vmin=-3, vmax=3)
ax[0, 2].imshow(decomp["cascade_levels"][1], cmap=cm.RdBu_r, vmin=-3, vmax=3)
ax[0, 3].imshow(decomp["cascade_levels"][2], cmap=cm.RdBu_r, vmin=-3, vmax=3)
ax[1, 0].imshow(decomp["cascade_levels"][3], cmap=cm.RdBu_r, vmin=-3, vmax=3)
from matplotlib import pyplot, ticker
import numpy as np
from scipy.interpolate import interp1d
from pysteps.cascade.bandpass_filters import filter_gaussian
domain = "fmi"
if domain == "fmi":
grid_size = 1226
else:
grid_size = 710
grid_res = 1.0
num_levels = 8
normalize = True
F = filter_gaussian(grid_size, num_levels, normalize=normalize)
fig = pyplot.figure(figsize=(5, 3.75))
ax1 = fig.gca()
ax2 = ax1.twiny()
w = F["weights_1d"]
cf = F["central_wavenumbers"]
for i in range(len(w)):
x_ip = np.linspace(0, len(w[i]) - 1, 10000)
y_ip = interp1d(np.arange(len(w[i])), w[i], kind="cubic")(x_ip)
ax1.semilogx(x_ip, y_ip, "k-", lw=2)
ax1.plot([cf[i], cf[i]], [0, 1], "k--")
cf[0] = 1
