def test_visualization_plot_precip_field( source, type, bbox, colorscale, probthr, title, colorbar, axis, ): if type == "intensity": field, metadata = get_precipitation_fields(0, 0, True, True, None, source) field = field.squeeze() field, metadata = conversion.to_rainrate(field, metadata) elif type == "depth": field, metadata = get_precipitation_fields(0, 0, True, True, None, source) field = field.squeeze() field, metadata = conversion.to_raindepth(field, metadata) elif type == "prob": field, metadata = get_precipitation_fields(0, 10, True, True, None, source) field, metadata = conversion.to_rainrate(field, metadata) field = ensemblestats.excprob(field, probthr) ax = plot_precip_field( field, type=type, bbox=bbox, geodata=None, colorscale=colorscale, probthr=probthr, units=metadata["unit"], title=title, colorbar=colorbar, axis=axis, )
def test_anvil_rainrate(n_cascade_levels, ar_order, ar_window_radius, timesteps, min_csi): """Tests ANVIL nowcast using rain rate precipitation fields.""" # inputs precip_input, metadata = get_precipitation_fields( num_prev_files=4, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] precip_obs = precip_obs.filled() pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) nowcast_method = nowcasts.get_method("anvil") precip_forecast = nowcast_method( precip_input[-(ar_order + 2):], retrieved_motion, timesteps=timesteps, rainrate=None, # no R(VIL) conversion is done n_cascade_levels=n_cascade_levels, ar_order=ar_order, ar_window_radius=ar_window_radius, ) assert precip_forecast.ndim == 3 assert precip_forecast.shape[0] == (timesteps if isinstance( timesteps, int) else len(timesteps)) result = verification.det_cat_fct(precip_forecast[-1], precip_obs[-1], thr=0.1, scores="CSI")["CSI"] assert result > min_csi, f"CSI={result:.2f}, required > {min_csi:.2f}"
def test_sseps(n_ens_members, n_cascade_levels, ar_order, mask_method, probmatching_method, win_size, max_crps): """Tests SSEPS nowcast.""" # inputs precip_input, metadata = get_precipitation_fields(num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000) precip_input = precip_input.filled() precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] precip_obs = precip_obs.filled() # Retrieve motion field pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) # Run nowcast nowcast_method = nowcasts.get_method("sseps") precip_forecast = nowcast_method( precip_input, metadata, retrieved_motion, win_size=win_size, n_timesteps=3, n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, ar_order=ar_order, seed=42, mask_method=mask_method, probmatching_method=probmatching_method, ) # result crps = verification.probscores.CRPS(precip_forecast[-1], precip_obs[-1]) print(f"got CRPS={crps:.1f}, required < {max_crps:.1f}") assert crps < max_crps
def test_sal_translation(self, converter, thr_factor): precip, metadata = get_precipitation_fields( num_prev_files=0, log_transform=False, metadata=True ) precip, metadata = converter(precip.filled(np.nan), metadata) precip_translated = np.roll(precip, 10, axis=0) result = sal(precip, precip_translated, thr_factor) assert np.allclose(result[0], 0) assert np.allclose(result[1], 0) assert not np.allclose(result[2], 0)
def test_sal_same_image(self, converter, thr_factor): """Test the SAL verification method.""" precip, metadata = get_precipitation_fields( num_prev_files=0, log_transform=False, metadata=True ) precip, metadata = converter(precip.filled(np.nan), metadata) result = sal(precip, precip, thr_factor) assert isinstance(result, tuple) assert len(result) == 3 assert np.allclose(result, [0, 0, 0])
def test_visualization_plot_precip_field(source, plot_type, bbox, colorscale, probthr, title, colorbar, axis): if plot_type == "intensity": field, metadata = get_precipitation_fields(0, 0, True, True, None, source) field = field.squeeze() field, metadata = conversion.to_rainrate(field, metadata) elif plot_type == "depth": field, metadata = get_precipitation_fields(0, 0, True, True, None, source) field = field.squeeze() field, metadata = conversion.to_raindepth(field, metadata) elif plot_type == "prob": field, metadata = get_precipitation_fields(0, 10, True, True, None, source) field, metadata = conversion.to_rainrate(field, metadata) field = ensemblestats.excprob(field, probthr) field_orig = field.copy() ax = plot_precip_field( field.copy(), ptype=plot_type, bbox=bbox, geodata=None, colorscale=colorscale, probthr=probthr, units=metadata["unit"], title=title, colorbar=colorbar, axis=axis, ) # Check that plot_precip_field does not modify the input data field_orig = np.ma.masked_invalid(field_orig) field_orig.data[field_orig.mask] = -100 field = np.ma.masked_invalid(field) field.data[field.mask] = -100 assert np.array_equal(field_orig.data, field.data)
def test_feature_tstorm_detection(source, output_feat, dry_input, max_num_features): pytest.importorskip("skimage") pytest.importorskip("pandas") if not dry_input: input, metadata = get_precipitation_fields(0, 0, True, True, None, source) input = input.squeeze() input, __ = to_reflectivity(input, metadata) else: input = np.zeros((50, 50)) time = "000" output = detection(input, time=time, output_feat=output_feat, max_num_features=max_num_features) if output_feat: assert isinstance(output, np.ndarray) assert output.ndim == 2 assert output.shape[1] == 2 if max_num_features is not None: assert output.shape[0] <= max_num_features else: assert isinstance(output, tuple) assert len(output) == 2 assert isinstance(output[0], DataFrame) assert isinstance(output[1], np.ndarray) if max_num_features is not None: assert output[0].shape[0] <= max_num_features assert output[0].shape[1] == 9 assert list(output[0].columns) == [ "ID", "time", "x", "y", "cen_x", "cen_y", "max_ref", "cont", "area", ] assert (output[0].time == time).all() assert output[1].ndim == 2 assert output[1].shape == input.shape if not dry_input: assert output[0].shape[0] > 0 assert sorted(list(output[0].ID)) == sorted( list(np.unique(output[1]))[1:]) else: assert output[0].shape[0] == 0 assert output[1].sum() == 0
def test_io_export_netcdf_one_member_one_time_step(): """ Test the export netcdf. Also, test that the exported file can be read by the importer. """ pytest.importorskip("pyproj") precip, metadata = get_precipitation_fields(return_raw=True, metadata=True, source="fmi") precip = precip.squeeze() invalid_mask = get_invalid_mask(precip) # save it back to disk with tempfile.TemporaryDirectory() as outpath: outfnprefix = "test_netcdf_out" file_path = os.path.join(outpath, outfnprefix + ".nc") startdate = metadata["timestamps"][0] timestep = metadata["accutime"] n_timesteps = 1 shape = precip.shape exporter = initialize_forecast_exporter_netcdf( outpath, outfnprefix, startdate, timestep, n_timesteps, shape, metadata, n_ens_members=1, ) export_forecast_dataset(precip[np.newaxis, :], exporter) close_forecast_files(exporter) # assert if netcdf file was saved and file size is not zero assert os.path.exists(file_path) and os.path.getsize(file_path) > 0 # Test that the file can be read by the nowcast_importer output_file_path = os.path.join(outpath, f"{outfnprefix}.nc") precip_new, _ = import_netcdf_pysteps(output_file_path) assert_array_almost_equal(precip, precip_new.data) assert precip_new.dtype == "single" precip_new, _ = import_netcdf_pysteps(output_file_path, dtype="double") assert_array_almost_equal(precip, precip_new.data) assert precip_new.dtype == "double" precip_new, _ = import_netcdf_pysteps(output_file_path, fillna=-1000) new_invalid_mask = precip_new == -1000 assert (new_invalid_mask == invalid_mask).all()
def test_feature(method, max_num_features): input_field, _ = get_precipitation_fields(0, 0, True, True, None, "mch") detector = feature.get_method(method) kwargs = {"max_num_features": max_num_features} output = detector(input_field.squeeze(), **kwargs) assert isinstance(output, np.ndarray) assert output.ndim == 2 assert output.shape[0] > 0 if max_num_features is not None: assert output.shape[0] <= max_num_features assert output.shape[1] == 2
def test_sal_zeros(self, converter, thr_factor): """Test the SAL verification method.""" precip, metadata = get_precipitation_fields( num_prev_files=0, log_transform=False, metadata=True ) precip, metadata = converter(precip.filled(np.nan), metadata) result = sal(precip * 0, precip * 0, thr_factor) assert np.isnan(result).all() result = sal(precip * 0, precip, thr_factor) assert result[:2] == (-2, -2) assert np.isnan(result[2]) result = sal(precip, precip * 0, thr_factor) assert result[:2] == (2, 2) assert np.isnan(result[2])
def test_visualization_plot_precip_field(source, map, drawlonlatlines, lw): field, metadata = get_precipitation_fields(0, 0, True, True, None, source) field = field.squeeze() field, __ = to_rainrate(field, metadata) ax = plot_precip_field( field, type="intensity", geodata=metadata, map=map, drawlonlatlines=drawlonlatlines, lw=lw, )
def test_visualization_plot_precip_field(source, map_kwargs, pass_geodata): field, metadata = get_precipitation_fields(0, 0, True, True, None, source) field = field.squeeze() field, __ = to_rainrate(field, metadata) if not pass_geodata: metadata = None ax = plot_precip_field( field, type="intensity", geodata=metadata, map_kwargs=map_kwargs, )
def test_visualization_motionfields_quiver( source, axis, step, quiver_kwargs, map_kwargs, upscale, pass_geodata, ): if source is not None: fields, geodata = get_precipitation_fields(0, 2, False, True, upscale, source) if not pass_geodata: geodata = None ax = plot_precip_field(fields[-1], geodata=geodata) oflow_method = motion.get_method("LK") UV = oflow_method(fields) else: shape = (100, 100) geodata = None ax = None u = np.ones(shape[1]) * shape[0] v = np.arange(0, shape[0]) U, V = np.meshgrid(u, v) UV = np.concatenate([U[None, :], V[None, :]]) UV_orig = UV.copy() __ = quiver( UV, ax, geodata, axis, step, quiver_kwargs, map_kwargs=map_kwargs, ) # Check that quiver does not modify the input data assert np.array_equal(UV, UV_orig)
def test_visualization_motionfields_streamplot( source, axis, streamplot_kwargs, map_kwargs, upscale, pass_geodata ): if source is not None: fields, geodata = get_precipitation_fields(0, 2, False, True, upscale, source) if not pass_geodata: pass_geodata = None ax = plot_precip_field(fields[-1], geodata=geodata) oflow_method = motion.get_method("LK") UV = oflow_method(fields) else: shape = (100, 100) geodata = None ax = None u = np.ones(shape[1]) * shape[0] v = np.arange(0, shape[0]) U, V = np.meshgrid(u, v) UV = np.concatenate([U[None, :], V[None, :]]) __ = streamplot(UV, ax, geodata, axis, streamplot_kwargs, map_kwargs=map_kwargs,)
def test_visualization_motionfields_quiver( source, map, drawlonlatlines, lw, axis, step, quiver_kwargs, upscale, ): if map == "cartopy": pytest.importorskip("cartopy") elif map == "basemap": pytest.importorskip("basemap") if source is not None: fields, geodata = get_precipitation_fields(0, 2, False, True, upscale, source) ax = plot_precip_field( fields[-1], map=map, geodata=geodata, ) oflow_method = motion.get_method("LK") UV = oflow_method(fields) else: shape = (100, 100) geodata = None ax = None u = np.ones(shape[1]) * shape[0] v = np.arange(0, shape[0]) U, V = np.meshgrid(u, v) UV = np.concatenate([U[None, :], V[None, :]]) __ = quiver(UV, ax, map, geodata, drawlonlatlines, lw, axis, step, quiver_kwargs)
def test_tracking_tdating_dating(source, dry_input): pytest.importorskip("skimage") pytest.importorskip("pandas") if not dry_input: input, metadata = get_precipitation_fields(0, 2, True, True, 4000, source) input, __ = to_reflectivity(input, metadata) else: input = np.zeros((3, 50, 50)) metadata = {"timestamps": ["00", "01", "02"]} timelist = metadata["timestamps"] output = dating(input, timelist, mintrack=1) # Check output format assert isinstance(output, tuple) assert len(output) == 3 assert isinstance(output[0], list) assert isinstance(output[1], list) assert isinstance(output[2], list) assert len(output[1]) == input.shape[0] assert len(output[2]) == input.shape[0] assert isinstance(output[1][0], DataFrame) assert isinstance(output[2][0], np.ndarray) assert output[1][0].shape[1] == 8 assert output[2][0].shape == input.shape[1:] if not dry_input: assert len(output[0]) > 0 assert isinstance(output[0][0], DataFrame) assert output[0][0].shape[1] == 8 else: assert len(output[0]) == 0 assert output[1][0].shape[0] == 0 assert output[2][0].sum() == 0
This tests check that the retrieved motion fields are within reasonable values. Also, they will fail if any modification on the code decrease the quality of the retrieval. """ import numpy as np import pytest from scipy.ndimage import uniform_filter import pysteps as stp from pysteps import motion from pysteps.motion.vet import morph from pysteps.tests.helpers import get_precipitation_fields, smart_assert reference_field = get_precipitation_fields(num_prev_files=0) def _create_motion_field(input_precip, motion_type): """ Create idealized motion fields to be applied to the reference image. Parameters ---------- input_precip: numpy array (lat, lon) motion_type : str The supported motion fields are: - linear_x: (u=2, v=0)
# -*- coding: utf-8 -*- import pytest from pysteps import downscaling from pysteps.tests.helpers import get_precipitation_fields from pysteps.utils import aggregate_fields_space, square_domain # load and preprocess input field precip, metadata = get_precipitation_fields( num_prev_files=0, num_next_files=0, return_raw=False, metadata=True, ) precip = precip.filled() precip, metadata = square_domain(precip, metadata, "crop") rainfarm_arg_names = ( "alpha", "ds_factor", "threshold", "return_alpha", ) rainfarm_arg_values = [ (1.0, 1, 0, False), (1, 2, 0, False), (1, 4, 0, False), ]
def test_steps_callback(tmp_path): """Test STEPS callback functionality to export the output as a netcdf.""" n_ens_members = 2 n_timesteps = 3 precip_input, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() field_shape = (precip_input.shape[1], precip_input.shape[2]) startdate = metadata["timestamps"][-1] timestep = metadata["accutime"] motion_field = np.zeros((2, *field_shape)) exporter = io.initialize_forecast_exporter_netcdf( outpath=tmp_path.as_posix(), outfnprefix="test_steps", startdate=startdate, timestep=timestep, n_timesteps=n_timesteps, shape=field_shape, n_ens_members=n_ens_members, metadata=metadata, incremental="timestep", ) def callback(array): return io.export_forecast_dataset(array, exporter) precip_output = nowcasts.get_method("steps")( precip_input, motion_field, timesteps=n_timesteps, R_thr=metadata["threshold"], kmperpixel=2.0, timestep=timestep, seed=42, n_ens_members=n_ens_members, vel_pert_method=None, callback=callback, return_output=True, ) io.close_forecast_files(exporter) # assert that netcdf exists and its size is not zero tmp_file = os.path.join(tmp_path, "test_steps.nc") assert os.path.exists(tmp_file) and os.path.getsize(tmp_file) > 0 # assert that the file can be read by the nowcast importer precip_netcdf, metadata_netcdf = io.import_netcdf_pysteps(tmp_file, dtype="float64") # assert that the dimensionality of the array is as expected assert precip_netcdf.ndim == 4, "Wrong number of dimensions" assert precip_netcdf.shape[0] == n_ens_members, "Wrong ensemble size" assert precip_netcdf.shape[1] == n_timesteps, "Wrong number of lead times" assert precip_netcdf.shape[2:] == field_shape, "Wrong field shape" # assert that the saved output is the same as the original output assert np.allclose( precip_netcdf, precip_output, equal_nan=True ), "Wrong output values" # assert that leadtimes and timestamps are as expected td = timedelta(minutes=timestep) leadtimes = [(i + 1) * timestep for i in range(n_timesteps)] timestamps = [startdate + (i + 1) * td for i in range(n_timesteps)] assert (metadata_netcdf["leadtimes"] == leadtimes).all(), "Wrong leadtimes" assert (metadata_netcdf["timestamps"] == timestamps).all(), "Wrong timestamps"
def test_lk( lk_kwargs, fd_method, dense, nr_std_outlier, k_outlier, size_opening, decl_scale, verbose, ): """Tests Lucas-Kanade optical flow.""" pytest.importorskip("cv2") if fd_method == "blob": pytest.importorskip("skimage") if fd_method == "tstorm": pytest.importorskip("skimage") pytest.importorskip("pandas") # inputs precip, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip = precip.filled() # Retrieve motion field oflow_method = motion.get_method("LK") output = oflow_method( precip, lk_kwargs=lk_kwargs, fd_method=fd_method, dense=dense, nr_std_outlier=nr_std_outlier, k_outlier=k_outlier, size_opening=size_opening, decl_scale=decl_scale, verbose=verbose, ) # Check format of ouput if dense: assert isinstance(output, np.ndarray) assert output.ndim == 3 assert output.shape[0] == 2 assert output.shape[1:] == precip[0].shape if nr_std_outlier == 0: assert output.sum() == 0 else: assert isinstance(output, tuple) assert len(output) == 2 assert isinstance(output[0], np.ndarray) assert isinstance(output[1], np.ndarray) assert output[0].ndim == 2 assert output[1].ndim == 2 assert output[0].shape[1] == 2 assert output[1].shape[1] == 2 assert output[0].shape[0] == output[1].shape[0] if nr_std_outlier == 0: assert output[0].shape[0] == 0 assert output[1].shape[0] == 0
# -*- coding: utf-8 -*- import numpy as np import pytest from numpy.testing import assert_array_almost_equal from pysteps.tests.helpers import get_precipitation_fields from pysteps.verification import ensscores precip = get_precipitation_fields(num_next_files=10, return_raw=True) np.random.seed(42) # rankhist test_data = [ (precip[:10], precip[-1], None, True, 11), (precip[:10], precip[-1], None, False, 11), ] @pytest.mark.parametrize("X_f, X_o, X_min, normalize, expected", test_data) def test_rankhist_size(X_f, X_o, X_min, normalize, expected): """Test the rankhist.""" assert_array_almost_equal( ensscores.rankhist(X_f, X_o, X_min, normalize).size, expected ) # ensemble_skill test_data = [ ( precip[:10],
# -*- coding: utf-8 -*- import pytest from numpy.testing import assert_array_almost_equal from pysteps.tests.helpers import get_precipitation_fields from pysteps.verification import spatialscores try: import pywt PYWT_IMPORTED = True except ImportError: PYWT_IMPORTED = False R = get_precipitation_fields(num_prev_files=1, return_raw=True) test_data = [(R[0], R[1], "FSS", [1], [10], None, 0.85161531)] if PYWT_IMPORTED: test_data.append((R[0], R[1], "BMSE", [1], None, "Haar", 0.99989651)) @pytest.mark.parametrize("X_f, X_o, name, thrs, scales, wavelet, expected", test_data) def test_intensity_scale(X_f, X_o, name, thrs, scales, wavelet, expected): """Test the intensity_scale.""" assert_array_almost_equal( spatialscores.intensity_scale(X_f, X_o, name, thrs, scales, wavelet)[0][0], expected, )
# -*- coding: utf-8 -*- import os import numpy as np import pytest from unittest.mock import patch from pysteps.tests.helpers import get_precipitation_fields from pysteps.visualization.animations import animate PRECIP, METADATA = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=True, metadata=True, upscale=2000, ) VALID_ARGS = ( ([PRECIP], {}), ([PRECIP], { "title": "title" }), ([PRECIP], { "timestamps_obs": METADATA["timestamps"] }), ([PRECIP], { "geodata": METADATA, "map_kwargs": { "plot_map": None
import numpy as np from pysteps.noise import fftgenerators from pysteps.tests.helpers import get_precipitation_fields PRECIP = get_precipitation_fields( num_prev_files=0, num_next_files=0, return_raw=False, metadata=False, upscale=2000, ) PRECIP = PRECIP.filled() def test_noise_param_2d_fft_filter(): fft_filter = fftgenerators.initialize_param_2d_fft_filter(PRECIP) assert isinstance(fft_filter, dict) assert all([ key in fft_filter for key in ["field", "input_shape", "model", "pars"] ]) out = fftgenerators.generate_noise_2d_fft_filter(fft_filter) assert isinstance(out, np.ndarray) assert out.shape == PRECIP.shape def test_noise_nonparam_2d_fft_filter():