def advection_correction(R, T=5, t=1): """ R = np.array([qpe_previous, qpe_current]) T = time between two observations (5 min) t = interpolation timestep (1 min) """ # Evaluate advection oflow_method = motion.get_method("LK") fd_kwargs = {"buffer_mask": 10} # avoid edge effects V = oflow_method(np.log(R), fd_kwargs=fd_kwargs) # Perform temporal interpolation Rd = np.zeros((R[0].shape)) x, y = np.meshgrid( np.arange(R[0].shape[1], dtype=float), np.arange(R[0].shape[0], dtype=float), ) for i in range(1, 1 + int(T / t)): pos1 = (y - i / T * V[1], x - i / T * V[0]) R1 = map_coordinates(R[0], pos1, order=1) pos2 = (y + (T - i) / T * V[1], x + (T - i) / T * V[0]) R2 = map_coordinates(R[1], pos2, order=1) Rd += (T - i) * R1 + i * R2 return 1 / T**2 * Rd
def test_steps_skill( n_ens_members, n_cascade_levels, ar_order, mask_method, probmatching_method, domain, timesteps, max_crps, ): """Tests STEPS nowcast skill.""" # inputs precip_input, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() precip_obs = get_precipitation_fields( num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000 )[1:, :, :] precip_obs = precip_obs.filled() pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) nowcast_method = nowcasts.get_method("steps") precip_forecast = nowcast_method( precip_input, retrieved_motion, timesteps=timesteps, R_thr=metadata["threshold"], kmperpixel=2.0, timestep=metadata["accutime"], seed=42, n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, ar_order=ar_order, mask_method=mask_method, probmatching_method=probmatching_method, domain=domain, ) assert precip_forecast.ndim == 4 assert precip_forecast.shape[0] == n_ens_members assert precip_forecast.shape[1] == ( timesteps if isinstance(timesteps, int) else len(timesteps) ) crps = verification.probscores.CRPS(precip_forecast[:, -1], precip_obs[-1]) assert crps < max_crps, f"CRPS={crps:.2f}, required < {max_crps:.2f}"
def test_steps( n_ens_members, n_cascade_levels, ar_order, mask_method, probmatching_method, domain, max_crps, ): """Tests STEPS nowcast.""" # inputs precip_input, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] precip_obs = precip_obs.filled() # Retrieve motion field pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) # Run nowcast nowcast_method = nowcasts.get_method("steps") precip_forecast = nowcast_method( precip_input, retrieved_motion, n_timesteps=3, R_thr=metadata["threshold"], kmperpixel=2.0, timestep=metadata["accutime"], seed=42, n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, ar_order=ar_order, mask_method=mask_method, probmatching_method=probmatching_method, domain=domain, ) # result crps = verification.probscores.CRPS(precip_forecast[-1], precip_obs[-1]) print(f"got CRPS={crps:.1f}, required < {max_crps:.1f}") assert crps < max_crps
def test_sprog( n_cascade_levels, ar_order, probmatching_method, domain, min_csi): """Tests SPROG nowcast.""" # inputs precip_input, metadata = get_precipitation_fields(num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000) precip_input = precip_input.filled() precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] precip_obs = precip_obs.filled() # Retrieve motion field pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) # Run nowcast nowcast_method = nowcasts.get_method("sprog") precip_forecast = nowcast_method( precip_input, retrieved_motion, n_timesteps=3, R_thr=metadata["threshold"], n_cascade_levels=n_cascade_levels, ar_order=ar_order, probmatching_method=probmatching_method, domain=domain, ) # result result = verification.det_cat_fct( precip_forecast[-1], precip_obs[-1], thr=0.1, scores="CSI")["CSI"] print(f"got CSI={result:.1f}, required > {min_csi:.1f}") assert result > min_csi
def DARTS(self, history, observation, th_dbz, n_leadtimes, n_frames=3, block_sizes=4, win_sizes=8, decl_scales=2): oflow_method = motion.get_method("DARTS") V = oflow_method(history) extrapolate = nowcasts.get_method("extrapolation") prediction_extrapolate = extrapolate(history[-1], V, n_leadtimes) return prediction_extrapolate
def test_steps( n_ens_members, n_cascade_levels, ar_order, mask_method, probmatching_method, max_crps): """Tests STEPS nowcast.""" # inputs precip_input, metadata = get_precipitation_fields(num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000) precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] # Retrieve motion field oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) # Run nowcast nowcast_method = nowcasts.get_method("steps") precip_forecast = nowcast_method( precip_input, retrieved_motion, n_timesteps=3, R_thr=metadata["threshold"], kmperpixel=2.0, timestep=metadata["accutime"], seed=42, n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, ar_order=ar_order, mask_method=mask_method, probmatching_method=probmatching_method, ) # result result = verification.probscores.CRPS(precip_forecast[-1], precip_obs[-1]) assert result < max_crps
def test_sprog( n_cascade_levels, ar_order, probmatching_method, domain, timesteps, min_csi ): """Tests SPROG nowcast.""" # inputs precip_input, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() precip_obs = get_precipitation_fields( num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000 )[1:, :, :] precip_obs = precip_obs.filled() pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) nowcast_method = nowcasts.get_method("sprog") precip_forecast = nowcast_method( precip_input, retrieved_motion, timesteps=timesteps, R_thr=metadata["threshold"], n_cascade_levels=n_cascade_levels, ar_order=ar_order, probmatching_method=probmatching_method, domain=domain, ) assert precip_forecast.ndim == 3 assert precip_forecast.shape[0] == ( timesteps if isinstance(timesteps, int) else len(timesteps) ) result = verification.det_cat_fct( precip_forecast[-1], precip_obs[-1], thr=0.1, scores="CSI" )["CSI"] assert result > min_csi, f"CSI={result:.1f}, required > {min_csi:.1f}"
def test_anvil_rainrate(n_cascade_levels, ar_order, ar_window_radius, timesteps, min_csi): """Tests ANVIL nowcast using rain rate precipitation fields.""" # inputs precip_input, metadata = get_precipitation_fields( num_prev_files=4, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] precip_obs = precip_obs.filled() pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) nowcast_method = nowcasts.get_method("anvil") precip_forecast = nowcast_method( precip_input[-(ar_order + 2):], retrieved_motion, timesteps=timesteps, rainrate=None, # no R(VIL) conversion is done n_cascade_levels=n_cascade_levels, ar_order=ar_order, ar_window_radius=ar_window_radius, ) assert precip_forecast.ndim == 3 assert precip_forecast.shape[0] == (timesteps if isinstance( timesteps, int) else len(timesteps)) result = verification.det_cat_fct(precip_forecast[-1], precip_obs[-1], thr=0.1, scores="CSI")["CSI"] assert result > min_csi, f"CSI={result:.2f}, required > {min_csi:.2f}"
def t_interp(t_boundary, img_boundary, analysis_time): """interpolate the two images temporally""" tstart2t0_weight, t02tend_weight = \ get_t_weights(analysis_time, t_boundary) dense_lucaskanade = motion.get_method('LK') uv_forward = dense_lucaskanade(img_boundary) uv_backward = dense_lucaskanade(img_boundary[::-1, :, :]) uv_forward, uv_backward = get_mean_uv(uv_forward, uv_backward) extrapolate = nowcasts.get_method('extrapolation') r_from_tstart = extrapolate(img_boundary[0, :, :], uv_forward * t02tend_weight, 1)[0, :, :] r_from_tend = extrapolate(img_boundary[1, :, :], uv_backward * tstart2t0_weight, 1)[0, :, :] r = r_from_tstart*tstart2t0_weight + \ r_from_tend*t02tend_weight return r
def getV(self, history, n_frame, block_size, win_size, decl_scale): oflow_method = motion.get_method("lucaskanade") fd_kwargs = {} fd_kwargs["max_corners"] = 1000 fd_kwargs["quality_level"] = 0.01 fd_kwargs["min_distance"] = 2 fd_kwargs["block_size"] = block_size # 4-8 lk_kwargs = {} lk_kwargs["winsize"] = (win_size, win_size) # 5-10 oflow_kwargs = {} oflow_kwargs["fd_kwargs"] = fd_kwargs oflow_kwargs["lk_kwargs"] = lk_kwargs oflow_kwargs["decl_scale"] = decl_scale # 2-4 V = oflow_method(history[-n_frame:], **oflow_kwargs) # 3 to 5 frames return V
def test_visualization_motionfields_quiver( source, axis, step, quiver_kwargs, map_kwargs, upscale, pass_geodata, ): if source is not None: fields, geodata = get_precipitation_fields(0, 2, False, True, upscale, source) if not pass_geodata: geodata = None ax = plot_precip_field(fields[-1], geodata=geodata) oflow_method = motion.get_method("LK") UV = oflow_method(fields) else: shape = (100, 100) geodata = None ax = None u = np.ones(shape[1]) * shape[0] v = np.arange(0, shape[0]) U, V = np.meshgrid(u, v) UV = np.concatenate([U[None, :], V[None, :]]) UV_orig = UV.copy() __ = quiver( UV, ax, geodata, axis, step, quiver_kwargs, map_kwargs=map_kwargs, ) # Check that quiver does not modify the input data assert np.array_equal(UV, UV_orig)
def test_no_precipitation(optflow_method_name, num_times): """ Test that the motion methods work well with a zero precipitation in the domain. The expected result is a zero motion vector. Parameters ---------- optflow_method_name: str Optical flow method name num_times : int Number of precipitation frames (times) used as input for the optical flow methods. """ zero_precip = np.zeros((num_times,) + reference_field.shape) motion_method = motion.get_method(optflow_method_name) uv_motion = motion_method(zero_precip, verbose=False) assert np.abs(uv_motion).max() < 0.01
def test_visualization_motionfields_streamplot( source, axis, streamplot_kwargs, map_kwargs, upscale, pass_geodata ): if source is not None: fields, geodata = get_precipitation_fields(0, 2, False, True, upscale, source) if not pass_geodata: pass_geodata = None ax = plot_precip_field(fields[-1], geodata=geodata) oflow_method = motion.get_method("LK") UV = oflow_method(fields) else: shape = (100, 100) geodata = None ax = None u = np.ones(shape[1]) * shape[0] v = np.arange(0, shape[0]) U, V = np.meshgrid(u, v) UV = np.concatenate([U[None, :], V[None, :]]) __ = streamplot(UV, ax, geodata, axis, streamplot_kwargs, map_kwargs=map_kwargs,)
def test_visualization_motionfields_quiver( source, map, drawlonlatlines, lw, axis, step, quiver_kwargs, upscale, ): if map == "cartopy": pytest.importorskip("cartopy") elif map == "basemap": pytest.importorskip("basemap") if source is not None: fields, geodata = get_precipitation_fields(0, 2, False, True, upscale, source) ax = plot_precip_field( fields[-1], map=map, geodata=geodata, ) oflow_method = motion.get_method("LK") UV = oflow_method(fields) else: shape = (100, 100) geodata = None ax = None u = np.ones(shape[1]) * shape[0] v = np.arange(0, shape[0]) U, V = np.meshgrid(u, v) UV = np.concatenate([U[None, :], V[None, :]]) __ = quiver(UV, ax, map, geodata, drawlonlatlines, lw, axis, step, quiver_kwargs)
def test_lk_masked_array(): """ Passing a ndarray with NaNs or a masked array should produce the same results. """ pytest.importorskip('cv2') __, precip_obs = _create_observations(reference_field.copy(), 'linear_y', num_times=2) motion_method = motion.get_method('LK') # ndarray with nans np.ma.set_fill_value(precip_obs, -15) ndarray = precip_obs.filled() ndarray[ndarray == -15] = np.nan uv_ndarray = motion_method(ndarray, fd_kwargs={'buffer_mask':20}, verbose=False) # masked array mdarray = np.ma.masked_invalid(ndarray) mdarray.data[mdarray.mask] = -15 uv_mdarray = motion_method(mdarray, fd_kwargs={'buffer_mask':20}, verbose=False) assert np.abs(uv_mdarray - uv_ndarray).max() < 0.01
def test_input_shape_checks(optflow_method_name, minimum_input_frames, maximum_input_frames): image_size = 100 motion_method = motion.get_method(optflow_method_name) if maximum_input_frames == np.inf: maximum_input_frames = minimum_input_frames + 10 with not_raises(Exception): for frames in range(minimum_input_frames, maximum_input_frames + 1): motion_method(np.zeros((frames, image_size, image_size)), verbose=False) with pytest.raises(ValueError): motion_method(np.zeros((2, ))) motion_method(np.zeros((2, 2))) for frames in range(minimum_input_frames): motion_method(np.zeros((frames, image_size, image_size)), verbose=False) for frames in range(maximum_input_frames + 1, maximum_input_frames + 4): motion_method(np.zeros((frames, image_size, image_size)), verbose=False)
def test_optflow_method_convergence(input_precip, optflow_method_name, motion_type, num_times, max_rel_rmse): """ Test the convergence to the actual solution of the optical flow method used. We measure the error in the retrieved field by using the Relative RMSE = Rel_RMSE = sqrt(Relative MSE) - Rel_RMSE = 0%: no error - Rel_RMSE = 100%: The retrieved motion field has an average error equal in magnitude to the motion field. Relative RMSE is computed only un a region surrounding the precipitation field, were we have enough information to retrieve the motion field. The precipitation region includes the precipitation pattern plus a margin of approximately 20 grid points. Parameters ---------- input_precip: numpy array (lat, lon) Input precipitation field. optflow_method_name: str Optical flow method name motion_type : str The supported motion fields are: - linear_x: (u=2, v=0) - linear_y: (u=0, v=2) """ ideal_motion, precip_obs = _create_observations(input_precip.copy(), motion_type, num_times=num_times) oflow_method = motion.get_method(optflow_method_name) if optflow_method_name == 'vet': # By default, the maximum number of iteration in the VET minimization # is maxiter=100. # To increase the stability of the tests to we increase this value to # maxiter=150. computed_motion = oflow_method(precip_obs, verbose=False, options=dict(maxiter=150)) else: computed_motion = oflow_method(precip_obs, verbose=False) precip_data, _ = stp.utils.dB_transform(precip_obs.max(axis=0), inverse=True) precip_data.data[precip_data.mask] = 0 precip_mask = ((uniform_filter(precip_data, size=20) > 0.1) & ~precip_obs.mask.any(axis=0)) # To evaluate the accuracy of the computed_motion vectors, we will use # a relative RMSE measure. # Relative MSE = < (expected_motion - computed_motion)^2 > / <expected_motion^2 > # Relative RMSE = sqrt(Relative MSE) mse = ((ideal_motion - computed_motion)[:, precip_mask] ** 2).mean() rel_mse = mse / (ideal_motion[:, precip_mask] ** 2).mean() rel_rmse = np.sqrt(rel_mse) * 100 print(f"method:{optflow_method_name} ; " f"motion:{motion_type} ; times: {num_times} ; " f"rel_rmse:{rel_rmse:.2f}%") assert rel_rmse < max_rel_rmse
# # Also note that the radar image includes NaNs in areas of missing data. # These are used by the optical flow algorithm to define the radar mask. # # Sparse Lucas-Kanade # ------------------- # # By setting the optional argument ``dense=False`` in ``xy, uv = dense_lucaskanade(...)``, # the LK algorithm returns the motion vectors detected by the Lucas-Kanade scheme # without interpolating them on the grid. # This allows us to better identify the presence of wrongly detected # stationary motion in areas where precipitation is leaving the domain (look # for the red dots within the blue circle in the figure below). # Get Lucas-Kanade optical flow method dense_lucaskanade = motion.get_method("LK") # Mask invalid values R = np.ma.masked_invalid(R) # Use no buffering of the radar mask fd_kwargs1 = {"buffer_mask": 0} xy, uv = dense_lucaskanade(R, dense=False, fd_kwargs=fd_kwargs1) plt.imshow(ref_dbr, cmap=plt.get_cmap("Greys")) plt.imshow(mask, cmap=colors.ListedColormap(["black"]), alpha=0.5) plt.quiver( xy[:, 0], xy[:, 1], uv[:, 0], uv[:, 1], color="red",
# Listing 2 from matplotlib import pyplot from pysteps import motion, nowcasts from pysteps.postprocessing.ensemblestats import excprob from pysteps.utils import transformation from pysteps.visualization import plot_precip_field # compute the advection field oflow_method = motion.get_method("lucaskanade") V = oflow_method(R) # compute the S-PROG nowcast nowcast_method = nowcasts.get_method("sprog") R_f_sprog = nowcast_method(R[-3:, :, :], V, 12, R_thr=-10.0)[-1, :, :] # compute the STEPS nowcast nowcast_method = nowcasts.get_method("steps") R_f = nowcast_method(R[-3:, :, :], V, 12, n_ens_members=24, n_cascade_levels=8, R_thr=-10.0, kmperpixel=1.0, timestep=5) # plot the S-PROG nowcast, one ensemble member of the STEPS nowcast and the exceedance # probability of 0.1 mm/h computed from the ensemble R_f_sprog = transformation.dB_transform(R_f_sprog, threshold=-10.0,
def compute(nowcast_method, config_number): # the optical flow methods to use oflow_methods = ["darts"] # time step between computation of each nowcast (minutes) timestep = 30 # the number of time steps for each nowcast (5 minutes for the MeteoSwiss and # FMI data) num_timesteps = 24 # the threshold to use for precipitation/no precipitation R_min = 0.1 R_min_dB = transformation.dB_transform(np.array([R_min]))[0][0] precip_events = [ ("201104160800", "201104170000"), ("201111152300", "201111161000"), ("201304110000", "201304120000"), ("201304041800", "201304051800"), ("201305180600", "201305181200"), ("201305270000", "201305271200"), ] for pei, pe in enumerate(precip_events): start_date = datetime.strptime(pe[0], "%Y%m%d%H%M") curdate = datetime.strptime(pe[0], "%Y%m%d%H%M") enddate = datetime.strptime(pe[1], "%Y%m%d%H%M") results = {} for m in oflow_methods: results[m] = {} results[m]["comptimes"] = 0.0 results[m]["CSI"] = [0.0] * num_timesteps results[m]["RMSE"] = [0.0] * num_timesteps results[m]["n_samples"] = [0.0] * num_timesteps my_observations = LowAltCompositeCollection() while curdate <= enddate: print("Computing nowcasts for event %d, start date %s..." % (pei + 1, str(curdate)), end="") sys.stdout.flush() if curdate + num_timesteps * timedelta(minutes=5) > enddate: break time_step_in_sec = 5 * 60 times = [ curdate - timedelta(seconds=time_step_in_sec * i) for i in range(9) ] times += [ curdate + timedelta(seconds=time_step_in_sec * i) for i in range(1, num_timesteps + 1) ] times.sort() # Add elements to the collection if they don't exists for _time in times: my_observations.add(get_lowalt_file(_time)) # First 9 times R = my_observations.get_data('Reflectivity', date=times[:9]) R = dbz_to_r(R, a=300., b=1.5) _R = list() # The original data is at 1km resolutions # Downscale to 5 km resolution by 5x5 averaging for i in range(9): _R.append(downscale_local_mean(R[i, :-1, :], (5, 5))) R = np.asarray(_R) my_observations.clean_buffers() # release memory missing_data = False for i in range(R.shape[0]): if not np.any(np.isfinite(R[i, :, :])): print("Skipping, no finite values found for time step %d" % (i + 1)) missing_data = True break if missing_data: curdate += timedelta(minutes=timestep) continue R = transformation.dB_transform(R)[0] # Forecast times fcts_times = times[9:] R_obs = my_observations.get_data('Reflectivity', date=times[9:]) R_obs = dbz_to_r(R_obs, a=300., b=1.5) # The original data is at 1km resolutions # Downscale to 5 km resolution by 5x5 averaging _R = list() for i in range(len(fcts_times)): _R.append(downscale_local_mean(R_obs[i, :-1, :], (5, 5))) R_obs = np.asarray(_R) my_observations.clean_buffers() # release memory for oflow_method in oflow_methods: oflow = motion.get_method(oflow_method) if oflow_method == "vet": R_ = R[-2:, :, :] else: R_ = R starttime = time.time() V = oflow(R_, **configurations[config_number]) print("%s optical flow computed in %.3f seconds." % \ (oflow_method, time.time() - starttime)) if nowcast_method == "advection": nc = nowcasts.get_method("extrapolation") R_fct = nc(R[-1, :, :], V, num_timesteps) else: nc = nowcasts.get_method("steps") R_fct = nc(R[-3:, :, :], V, num_timesteps, noise_method=None, vel_pert_method=None, n_ens_members=1, mask_method="sprog", R_thr=R_min_dB, probmatching_method="mean", fft_method="numpy")[0, :, :, :] R_fct = transformation.dB_transform(R_fct, inverse=True)[0] for lt in range(num_timesteps): if not np.any(np.isfinite(R_obs[lt, :, :])): print( "Warning: no finite verifying observations for lead time %d." % (lt + 1)) continue csi = det_cat_fcst(R_fct[lt, :, :], R_obs[lt, :, :], R_min, ["CSI"])[0] MASK = np.logical_and(R_fct[lt, :, :] > R_min, R_obs[lt, :, :] > R_min) if np.sum(MASK) == 0: print("Skipping, no precipitation for lead time %d." % (lt + 1)) continue rmse = det_cont_fcst(R_fct[lt, :, :][MASK], R_obs[lt, :, :][MASK], ["MAE_add"])[0] results[oflow_method]["CSI"][lt] += csi results[oflow_method]["RMSE"][lt] += rmse results[oflow_method]["n_samples"][lt] += 1 print("Done.") curdate += timedelta(minutes=timestep) data_dir = './data/dart_tests/config_{:d}'.format(config_number) create_dir(data_dir) file_name = "optflow_comparison_results_%s_%s.dat" % ( nowcast_method, get_timestamp(start_date)) with open(join(data_dir, file_name), "wb") as f: pickle.dump(results, f)
def test_lk( lk_kwargs, fd_method, dense, nr_std_outlier, k_outlier, size_opening, decl_scale, verbose, ): """Tests Lucas-Kanade optical flow.""" pytest.importorskip("cv2") if fd_method == "blob": pytest.importorskip("skimage") if fd_method == "tstorm": pytest.importorskip("skimage") pytest.importorskip("pandas") # inputs precip, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip = precip.filled() # Retrieve motion field oflow_method = motion.get_method("LK") output = oflow_method( precip, lk_kwargs=lk_kwargs, fd_method=fd_method, dense=dense, nr_std_outlier=nr_std_outlier, k_outlier=k_outlier, size_opening=size_opening, decl_scale=decl_scale, verbose=verbose, ) # Check format of ouput if dense: assert isinstance(output, np.ndarray) assert output.ndim == 3 assert output.shape[0] == 2 assert output.shape[1:] == precip[0].shape if nr_std_outlier == 0: assert output.sum() == 0 else: assert isinstance(output, tuple) assert len(output) == 2 assert isinstance(output[0], np.ndarray) assert isinstance(output[1], np.ndarray) assert output[0].ndim == 2 assert output[1].ndim == 2 assert output[0].shape[1] == 2 assert output[1].shape[1] == 2 assert output[0].shape[0] == output[1].shape[0] if nr_std_outlier == 0: assert output[0].shape[0] == 0 assert output[1].shape[0] == 0
# Store the last frame for plotting it later later R_ = R[-1, :, :].copy() # Log-transform the data to unit of dBR, set the threshold to 0.1 mm/h, # set the fill value to -15 dBR R, metadata = transformation.dB_transform(R, metadata, threshold=0, zerovalue=-15.0) # Nicely print the metadata pprint(metadata) # Estimate the motion field with Lucas-Kanade oflow_method = motion.get_method("LK") V = oflow_method(R[-3:, :, :]) # Extrapolate the last radar observation extrapolate = nowcasts.get_method("extrapolation") R[~np.isfinite(R)] = metadata["zerovalue"] R_f = extrapolate(R[-1, :, :], V, n_leadtimes) # Back-transform to rain rate R_f = transformation.dB_transform(R_f, threshold=0.1, inverse=True)[0] # Plot the motion field animations.animate(R, nloops=1, R_fct=R_f, timestep_min=5,
# # Also note that the radar image includes NaNs in areas of missing data. # These are used by the optical flow algorithm to define the radar mask. # # Sparse Lucas-Kanade # ------------------- # # By setting the optional argument 'dense=False' in 'x,y,u,v = LK_optflow(.....)', # the LK algorithm returns the motion vectors detected by the Lucas-Kanade scheme # without interpolating them on the grid. # This allows us to better identify the presence of wrongly detected # stationary motion in areas where precipitation is leaving the domain (look # for the red dots within the blue circle in the figure below). # get Lucas-Kanade optical flow method LK_optflow = motion.get_method("LK") # Mask invalid values R = np.ma.masked_invalid(R) R.data[R.mask] = np.nan # Use default settings (i.e., no buffering of the radar mask) x, y, u, v = LK_optflow(R, dense=False, buffer_mask=0, quality_level_ST=0.1) plt.imshow(ref_dbr, cmap=plt.get_cmap("Greys")) plt.imshow(mask, cmap=colors.ListedColormap(["black"]), alpha=0.5) plt.quiver(x, y, u, v, color="red", angles="xy", scale_units="xy", scale=0.2) circle = plt.Circle((620, 245), 100, color="b", clip_on=False, fill=False) plt.gca().add_artist(circle) plt.title("buffer_mask = 0 (default)") plt.show()
zerovalue=-15.0) # Nicely print the metadata pprint(metadata) ################################################################################ # Lucas-Kanade (LK) # ----------------- # # The Lucas-Kanade optical flow method implemented in pysteps is a local # tracking approach that relies on the OpenCV package. # Local features are tracked in a sequence of two or more radar images. The # scheme includes a final interpolation step in order to produce a smooth # field of motion vectors. oflow_method = motion.get_method("LK") V1 = oflow_method(R[-3:, :, :]) # Plot the motion field on top of the reference frame plot_precip_field(R_, geodata=metadata, title="LK") quiver(V1, geodata=metadata, step=25) plt.show() ################################################################################ # Variational echo tracking (VET) # ------------------------------- # # This module implements the VET algorithm presented # by Laroche and Zawadzki (1995) and used in the McGill Algorithm for # Prediction by Lagrangian Extrapolation (MAPLE) described in # Germann and Zawadzki (2002).
def dating( input_video, timelist, mintrack=3, cell_list=None, label_list=None, start=0, minref=35, maxref=48, mindiff=6, minsize=50, minmax=41, mindis=10, dyn_thresh=False, ): """ This function performs the thunderstorm detection and tracking DATing. It requires a 3-D input array that contains the temporal succession of the 2-D data array of each timestep. On each timestep the detection is performed, the identified objects are advected with a flow prediction and the advected objects are matched to the newly identified objects of the next timestep. The last portion re-arranges the data into tracks sorted by ID-number. Parameters ---------- input_video : array-like Array of shape (t,m,n) containing input image, with t being the temporal dimension and m,n the spatial dimensions. Thresholds are tuned to maximum reflectivity in dBZ with a spatial resolution of 1 km and a temporal resolution of 5 min. Nan values are ignored. timelist : list List of length t containing string of time and date of each (m,n) field. mintrack : int, optional minimum duration of cell-track to be counted. The default is 3 time steps. cell_list : list or None, optional If you wish to expand an existing list of cells, insert previous cell-list here. The default is None. If not None, requires that label_list has the same length. label_list : list or None, optional If you wish to expand an existing list of cells, insert previous label-list here. The default is None. If not None, requires that cell_list has the same length. start : int, optional If you wish to expand an existing list of cells, the input video must contain 2 timesteps prior to the merging. The start can then be set to 2, allowing the motion vectors to be formed from the first three grids and continuing the cell tracking from there. The default is 0, which initiates a new tracking sequence. minref : float, optional Lower threshold for object detection. Lower values will be set to NaN. The default is 35 dBZ. maxref : float, optional Upper threshold for object detection. Higher values will be set to this value. The default is 48 dBZ. mindiff : float, optional Minimal difference between two identified maxima within same area to split area into two objects. The default is 6 dBZ. minsize : float, optional Minimal area for possible detected object. The default is 50 pixels. minmax : float, optional Minimum value of maximum in identified objects. Objects with a maximum lower than this will be discarded. The default is 41 dBZ. mindis : float, optional Minimum distance between two maxima of identified objects. Objects with a smaller distance will be merged. The default is 10 km. Returns ------- track_list : list of dataframes Each dataframe contains the track and properties belonging to one cell ID. Columns of dataframes: ID - cell ID, time - time stamp, x - array of all x-coordinates of cell, y - array of all y-coordinates of cell, cen_x - x-coordinate of cell centroid, cen_y - y-coordinate of cell centroid, max_ref - maximum (reflectivity) value of cell, cont - cell contours cell_list : list of dataframes Each dataframe contains the detected cells and properties belonging to one timestep. The IDs are already matched to provide a track. Columns of dataframes: ID - cell ID, time - time stamp, x - array of all x-coordinates of cell, y - array of all y-coordinates of cell, cen_x - x-coordinate of cell centroid, cen_y - y-coordinate of cell centroid, max_ref - maximum (reflectivity) value of cell, cont - cell contours label_list : list of arrays Each (n,m) array contains the gridded IDs of the cells identified in the corresponding timestep. The IDs are already matched to provide a track. """ if not SKIMAGE_IMPORTED: raise MissingOptionalDependency( "skimage is required for thunderstorm DATing " "but it is not installed") if not PANDAS_IMPORTED: raise MissingOptionalDependency( "pandas is required for thunderstorm DATing " "but it is not installed") # Check arguments if cell_list is None or label_list is None: cell_list = [] label_list = [] else: if not len(cell_list) == len(label_list): raise ValueError("len(cell_list) != len(label_list)") if start > len(timelist): raise ValueError("start > len(timelist)") oflow_method = motion.get_method("LK") max_ID = 0 for t in range(start, len(timelist)): cells_id, labels = tstorm_detect.detection( input_video[t, :, :], minref=minref, maxref=maxref, mindiff=mindiff, minsize=minsize, minmax=minmax, mindis=mindis, time=timelist[t], ) if len(cell_list) < 2: cell_list.append(cells_id) label_list.append(labels) cid = np.unique(labels) max_ID = np.nanmax([np.nanmax(cid), max_ID]) + 1 continue if t >= 2: flowfield = oflow_method(input_video[t - 2:t + 1, :, :]) cells_id, max_ID, newlabels = tracking(cells_id, cell_list[-1], labels, flowfield, max_ID) cid = np.unique(newlabels) # max_ID = np.nanmax([np.nanmax(cid), max_ID]) cell_list.append(cells_id) label_list.append(newlabels) track_list = couple_track(cell_list[2:], int(max_ID), mintrack) return track_list, cell_list, label_list
zerovalue=-15.0) # Nicely print the metadata pprint(metadata) ################################################################################ # Lucas-Kanade (LK) # ----------------- # # The Lucas-Kanade optical flow method implemented in pysteps is a local # tracking approach that relies on the OpenCV package. # Local features are tracked in a sequence of two or more radar images. The # scheme includes a final interpolation step in order to produce a smooth # field of motion vectors. oflow_method = motion.get_method("LK") V1 = oflow_method(R[-3:, :, :]) # Plot the motion field on top of the reference frame plot_precip_field(R_, geodata=metadata, title="LK") quiver(V1, geodata=metadata, step=25) plt.show() ################################################################################ # Variational echo tracking (VET) # ------------------------------- # # This module implements the VET algorithm presented # by Laroche and Zawadzki (1995) and used in the McGill Algorithm for # Prediction by Lagrangian Extrapolation (MAPLE) described in # Germann and Zawadzki (2002).
nargs='?', type=str, metavar="filename", help="accumulate statistics to previously computed file <filename>") args = argparser.parse_args() datasource = rcparams["data_sources"][args.datasource] startdate = datetime.strptime(args.startdate, "%Y%m%d%H%M") enddate = datetime.strptime(args.enddate, "%Y%m%d%H%M") importer = io.get_method(datasource["importer"], "importer") motionfields = {} oflow = motion.get_method(args.oflow) # compute motion fields # --------------------- # TODO: This keeps all motion fields in memory during the analysis period, which # can take a lot of memory. curdate = startdate while curdate <= enddate: try: fns = io.archive.find_by_date(curdate, datasource["root_path"], datasource["path_fmt"], datasource["fn_pattern"], datasource["fn_ext"],
def plot_optflow_method_convergence(input_precip, optflow_method_name, motion_type): """ Test the convergence to the actual solution of the optical flow method used. Parameters ---------- input_precip: numpy array (lat, lon) Input precipitation field. optflow_method_name: str Optical flow method name motion_type : str The supported motion fields are: - linear_x: (u=2, v=0) - linear_y: (u=0, v=2) - rotor: rotor field """ if optflow_method_name.lower() != "darts": num_times = 2 else: num_times = 9 ideal_motion, precip_obs = create_observations(input_precip, motion_type, num_times=num_times) oflow_method = motion.get_method(optflow_method_name) computed_motion = oflow_method(precip_obs, verbose=False) precip_obs, _ = stp.utils.dB_transform(precip_obs, inverse=True) precip_data = precip_obs.max(axis=0) precip_data.data[precip_data.mask] = 0 precip_mask = ((uniform_filter(precip_data, size=20) > 0.1) & ~precip_obs.mask.any(axis=0)) cmap = get_cmap('jet') cmap.set_under('grey', alpha=0.25) cmap.set_over('none') # Compare retrieved motion field with the ideal one plt.figure(figsize=(9, 4)) plt.subplot(1, 2, 1) ax = plot_precip_field(precip_obs[0], title="Reference motion") quiver(ideal_motion, step=25, ax=ax) plt.subplot(1, 2, 2) ax = plot_precip_field(precip_obs[0], title="Retrieved motion") quiver(computed_motion, step=25, ax=ax) # To evaluate the accuracy of the computed_motion vectors, we will use # a relative RMSE measure. # Relative MSE = < (expected_motion - computed_motion)^2 > / <expected_motion^2 > # Relative RMSE = sqrt(Relative MSE) mse = ((ideal_motion - computed_motion)[:, precip_mask] ** 2).mean() rel_mse = mse / (ideal_motion[:, precip_mask] ** 2).mean() plt.suptitle(f"{optflow_method_name} " f"Relative RMSE: {np.sqrt(rel_mse) * 100:.2f}%") plt.show()
results = {} results["leadtimes"] = (np.arange(num_timesteps) + 1) * datasource["timestep"] results["timestep"] = datasource["timestep"] results["cc_ar"] = [np.zeros(num_timesteps) for i in range(num_cascade_levels)] results["cc_obs"] = [ np.zeros(num_timesteps) for i in range(num_cascade_levels) ] results["n_ar_samples"] = [np.zeros(num_timesteps, dtype=int) \ for i in range(num_cascade_levels)] results["n_obs_samples"] = [np.zeros(num_timesteps, dtype=int) \ for i in range(num_cascade_levels)] filter = None oflow = motion.get_method(oflow_method) extrapolator = extrapolation.get_method("semilagrangian") for pei, pe in enumerate(precipevents): curdate = datetime.strptime(pe[0], "%Y%m%d%H%M") enddate = datetime.strptime(pe[1], "%Y%m%d%H%M") while curdate <= enddate: print("Running analyses for event %d, start date %s..." % (pei + 1, str(curdate)), end="") sys.stdout.flush() if curdate + num_timesteps * timedelta(minutes=5) > enddate: print("Done.") break