def S_PROG(self, history, observation, th_dbz, n_leadtimes, n_frames=3, block_sizes=7, win_sizes=8, decl_scales=3, n_cascade_levels=3): V = self.getV(history, n_frames, block_sizes, win_sizes, decl_scales) th = th_dbz / 70 * 255 sprog = nowcasts.get_method("sprog") prediction_sprog = sprog( history, # it takes last three V, n_leadtimes, n_cascade_levels=n_cascade_levels, # 3-4 R_thr=th, # reflectivity th decomp_method="fft", bandpass_filter_method="gaussian", probmatching_method="mean", ) return prediction_sprog
def test_steps( n_ens_members, n_cascade_levels, ar_order, mask_method, probmatching_method, max_crps, ): """Tests STEPS nowcast.""" # inputs R, metadata = _import_mch_gif(2, 0) R_o = _import_mch_gif(0, 3)[0][1:, :, :] # optical flow of_method = motion.get_method("LK") V = of_method(R) # nowcast nowcast_method = nowcasts.get_method("steps") num_timesteps = 1 R_f = nowcast_method( R, V, n_timesteps=3, R_thr=metadata["threshold"], kmperpixel=2.0, timestep=metadata["accutime"], seed=42, n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, ar_order=ar_order, mask_method=mask_method, probmatching_method=probmatching_method, ) # result result = verification.probscores.CRPS(R_f[-1], R_o[-1]) assert result < max_crps
def test_steps_skill( n_ens_members, n_cascade_levels, ar_order, mask_method, probmatching_method, domain, timesteps, max_crps, ): """Tests STEPS nowcast skill.""" # inputs precip_input, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() precip_obs = get_precipitation_fields( num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000 )[1:, :, :] precip_obs = precip_obs.filled() pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) nowcast_method = nowcasts.get_method("steps") precip_forecast = nowcast_method( precip_input, retrieved_motion, timesteps=timesteps, R_thr=metadata["threshold"], kmperpixel=2.0, timestep=metadata["accutime"], seed=42, n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, ar_order=ar_order, mask_method=mask_method, probmatching_method=probmatching_method, domain=domain, ) assert precip_forecast.ndim == 4 assert precip_forecast.shape[0] == n_ens_members assert precip_forecast.shape[1] == ( timesteps if isinstance(timesteps, int) else len(timesteps) ) crps = verification.probscores.CRPS(precip_forecast[:, -1], precip_obs[-1]) assert crps < max_crps, f"CRPS={crps:.2f}, required < {max_crps:.2f}"
def test_steps( n_ens_members, n_cascade_levels, ar_order, mask_method, probmatching_method, domain, max_crps, ): """Tests STEPS nowcast.""" # inputs precip_input, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] precip_obs = precip_obs.filled() # Retrieve motion field pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) # Run nowcast nowcast_method = nowcasts.get_method("steps") precip_forecast = nowcast_method( precip_input, retrieved_motion, n_timesteps=3, R_thr=metadata["threshold"], kmperpixel=2.0, timestep=metadata["accutime"], seed=42, n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, ar_order=ar_order, mask_method=mask_method, probmatching_method=probmatching_method, domain=domain, ) # result crps = verification.probscores.CRPS(precip_forecast[-1], precip_obs[-1]) print(f"got CRPS={crps:.1f}, required < {max_crps:.1f}") assert crps < max_crps
def extrapolation(self, history, observation, th_dbz, n_leadtimes, n_frames=3, block_sizes=4, win_sizes=8, decl_scales=2): V = self.getV(history, n_frames, block_sizes, win_sizes, decl_scales) extrapolate = nowcasts.get_method("extrapolation") prediction_extrapolate = extrapolate(history[-1], V, n_leadtimes) return prediction_extrapolate
def test_sprog( n_cascade_levels, ar_order, probmatching_method, domain, min_csi): """Tests SPROG nowcast.""" # inputs precip_input, metadata = get_precipitation_fields(num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000) precip_input = precip_input.filled() precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] precip_obs = precip_obs.filled() # Retrieve motion field pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) # Run nowcast nowcast_method = nowcasts.get_method("sprog") precip_forecast = nowcast_method( precip_input, retrieved_motion, n_timesteps=3, R_thr=metadata["threshold"], n_cascade_levels=n_cascade_levels, ar_order=ar_order, probmatching_method=probmatching_method, domain=domain, ) # result result = verification.det_cat_fct( precip_forecast[-1], precip_obs[-1], thr=0.1, scores="CSI")["CSI"] print(f"got CSI={result:.1f}, required > {min_csi:.1f}") assert result > min_csi
def STEPS(self, history, observation, th_dbz, n_leadtimes, n_frames=4, block_sizes=6, win_sizes=6, decl_scales=3, n_cascade_levels=3, ar_window_radius=None): V = self.getV(history, n_frames, block_sizes, win_sizes, decl_scales) if observation is not None: n_ens_members = 20 else: n_ens_members = 100 th = th_dbz / 70 * 255 steps = nowcasts.get_method("steps") prediction_steps = steps( history, # last 3 V, n_leadtimes, n_ens_members=20, n_cascade_levels=n_cascade_levels, # 3-4 R_thr=th, # Z th kmperpixel=4.6875, # pixel resolution in km 64/300=4.6875 timestep=5, decomp_method="fft", bandpass_filter_method="gaussian", noise_method="nonparametric", vel_pert_method="bps", mask_method="incremental", seed=24, ) #Two options: get the best prediction or do the average (for the avg use 50 or 100 samples) if observation is not None: best_ETS = -2 for p in prediction_steps: ETS = self.getETS(observation, p, th_dbz=th_dbz) if ETS > best_ETS: best_ETS = ETS best_prediction = p else: best_prediction = prediction_steps.mean(axis=0) return best_prediction
def DARTS(self, history, observation, th_dbz, n_leadtimes, n_frames=3, block_sizes=4, win_sizes=8, decl_scales=2): oflow_method = motion.get_method("DARTS") V = oflow_method(history) extrapolate = nowcasts.get_method("extrapolation") prediction_extrapolate = extrapolate(history[-1], V, n_leadtimes) return prediction_extrapolate
def test_sprog( n_cascade_levels, ar_order, probmatching_method, domain, timesteps, min_csi ): """Tests SPROG nowcast.""" # inputs precip_input, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() precip_obs = get_precipitation_fields( num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000 )[1:, :, :] precip_obs = precip_obs.filled() pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) nowcast_method = nowcasts.get_method("sprog") precip_forecast = nowcast_method( precip_input, retrieved_motion, timesteps=timesteps, R_thr=metadata["threshold"], n_cascade_levels=n_cascade_levels, ar_order=ar_order, probmatching_method=probmatching_method, domain=domain, ) assert precip_forecast.ndim == 3 assert precip_forecast.shape[0] == ( timesteps if isinstance(timesteps, int) else len(timesteps) ) result = verification.det_cat_fct( precip_forecast[-1], precip_obs[-1], thr=0.1, scores="CSI" )["CSI"] assert result > min_csi, f"CSI={result:.1f}, required > {min_csi:.1f}"
def test_steps( n_ens_members, n_cascade_levels, ar_order, mask_method, probmatching_method, max_crps): """Tests STEPS nowcast.""" # inputs precip_input, metadata = get_precipitation_fields(num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000) precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] # Retrieve motion field oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) # Run nowcast nowcast_method = nowcasts.get_method("steps") precip_forecast = nowcast_method( precip_input, retrieved_motion, n_timesteps=3, R_thr=metadata["threshold"], kmperpixel=2.0, timestep=metadata["accutime"], seed=42, n_ens_members=n_ens_members, n_cascade_levels=n_cascade_levels, ar_order=ar_order, mask_method=mask_method, probmatching_method=probmatching_method, ) # result result = verification.probscores.CRPS(precip_forecast[-1], precip_obs[-1]) assert result < max_crps
def test_anvil_rainrate(n_cascade_levels, ar_order, ar_window_radius, timesteps, min_csi): """Tests ANVIL nowcast using rain rate precipitation fields.""" # inputs precip_input, metadata = get_precipitation_fields( num_prev_files=4, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() precip_obs = get_precipitation_fields(num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000)[1:, :, :] precip_obs = precip_obs.filled() pytest.importorskip("cv2") oflow_method = motion.get_method("LK") retrieved_motion = oflow_method(precip_input) nowcast_method = nowcasts.get_method("anvil") precip_forecast = nowcast_method( precip_input[-(ar_order + 2):], retrieved_motion, timesteps=timesteps, rainrate=None, # no R(VIL) conversion is done n_cascade_levels=n_cascade_levels, ar_order=ar_order, ar_window_radius=ar_window_radius, ) assert precip_forecast.ndim == 3 assert precip_forecast.shape[0] == (timesteps if isinstance( timesteps, int) else len(timesteps)) result = verification.det_cat_fct(precip_forecast[-1], precip_obs[-1], thr=0.1, scores="CSI")["CSI"] assert result > min_csi, f"CSI={result:.2f}, required > {min_csi:.2f}"
def t_interp(t_boundary, img_boundary, analysis_time): """interpolate the two images temporally""" tstart2t0_weight, t02tend_weight = \ get_t_weights(analysis_time, t_boundary) dense_lucaskanade = motion.get_method('LK') uv_forward = dense_lucaskanade(img_boundary) uv_backward = dense_lucaskanade(img_boundary[::-1, :, :]) uv_forward, uv_backward = get_mean_uv(uv_forward, uv_backward) extrapolate = nowcasts.get_method('extrapolation') r_from_tstart = extrapolate(img_boundary[0, :, :], uv_forward * t02tend_weight, 1)[0, :, :] r_from_tend = extrapolate(img_boundary[1, :, :], uv_backward * tstart2t0_weight, 1)[0, :, :] r = r_from_tstart*tstart2t0_weight + \ r_from_tend*t02tend_weight return r
def ANVIL(self, history, observation, th_dbz, n_leadtimes, n_frames=3, block_sizes=6, win_sizes=9, decl_scales=3, n_cascade_levels=23, ar_window_radius=3): V = self.getV(history, n_frames, block_sizes, win_sizes, decl_scales) anvil = nowcasts.get_method("anvil") prediction_anvil = anvil( history[-3:], V, n_leadtimes, n_cascade_levels=n_cascade_levels, ar_window_radius=ar_window_radius, # half of domain 15-40 ar_order=1) return prediction_anvil
################################################################################ # Notice how the presence of erroneous velocity vectors produces a significantly # slower motion field near the right edge of the domain. # # Forecast skill # -------------- # # We are now going to evaluate the benefit of buffering the radar mask by computing # the forecast skill in terms of the Spearman correlation coefficient. # The extrapolation forecasts are computed using the dense UV motion fields # estimated above. # Get the advection routine and extrapolate the last radar frame by 12 time steps # (i.e., 1 hour lead time) extrapolate = nowcasts.get_method("extrapolation") R[~np.isfinite(R)] = metadata["zerovalue"] R_f1 = extrapolate(R[-1], UV1, 12) R_f2 = extrapolate(R[-1], UV2, 12) # Back-transform to rain rate R_f1 = transformation.dB_transform(R_f1, threshold=-10.0, inverse=True)[0] R_f2 = transformation.dB_transform(R_f2, threshold=-10.0, inverse=True)[0] # Find the veriyfing observations in the archive fns = io.archive.find_by_date(date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5,
# Listing 2 from matplotlib import pyplot from pysteps import motion, nowcasts from pysteps.postprocessing.ensemblestats import excprob from pysteps.utils import transformation from pysteps.visualization import plot_precip_field # compute the advection field oflow_method = motion.get_method("lucaskanade") V = oflow_method(R) # compute the S-PROG nowcast nowcast_method = nowcasts.get_method("sprog") R_f_sprog = nowcast_method(R[-3:, :, :], V, 12, R_thr=-10.0)[-1, :, :] # compute the STEPS nowcast nowcast_method = nowcasts.get_method("steps") R_f = nowcast_method(R[-3:, :, :], V, 12, n_ens_members=24, n_cascade_levels=8, R_thr=-10.0, kmperpixel=1.0, timestep=5) # plot the S-PROG nowcast, one ensemble member of the STEPS nowcast and the exceedance # probability of 0.1 mm/h computed from the ensemble R_f_sprog = transformation.dB_transform(R_f_sprog, threshold=-10.0,
def test_steps_callback(tmp_path): """Test STEPS callback functionality to export the output as a netcdf.""" n_ens_members = 2 n_timesteps = 3 precip_input, metadata = get_precipitation_fields( num_prev_files=2, num_next_files=0, return_raw=False, metadata=True, upscale=2000, ) precip_input = precip_input.filled() field_shape = (precip_input.shape[1], precip_input.shape[2]) startdate = metadata["timestamps"][-1] timestep = metadata["accutime"] motion_field = np.zeros((2, *field_shape)) exporter = io.initialize_forecast_exporter_netcdf( outpath=tmp_path.as_posix(), outfnprefix="test_steps", startdate=startdate, timestep=timestep, n_timesteps=n_timesteps, shape=field_shape, n_ens_members=n_ens_members, metadata=metadata, incremental="timestep", ) def callback(array): return io.export_forecast_dataset(array, exporter) precip_output = nowcasts.get_method("steps")( precip_input, motion_field, timesteps=n_timesteps, R_thr=metadata["threshold"], kmperpixel=2.0, timestep=timestep, seed=42, n_ens_members=n_ens_members, vel_pert_method=None, callback=callback, return_output=True, ) io.close_forecast_files(exporter) # assert that netcdf exists and its size is not zero tmp_file = os.path.join(tmp_path, "test_steps.nc") assert os.path.exists(tmp_file) and os.path.getsize(tmp_file) > 0 # assert that the file can be read by the nowcast importer precip_netcdf, metadata_netcdf = io.import_netcdf_pysteps(tmp_file, dtype="float64") # assert that the dimensionality of the array is as expected assert precip_netcdf.ndim == 4, "Wrong number of dimensions" assert precip_netcdf.shape[0] == n_ens_members, "Wrong ensemble size" assert precip_netcdf.shape[1] == n_timesteps, "Wrong number of lead times" assert precip_netcdf.shape[2:] == field_shape, "Wrong field shape" # assert that the saved output is the same as the original output assert np.allclose( precip_netcdf, precip_output, equal_nan=True ), "Wrong output values" # assert that leadtimes and timestamps are as expected td = timedelta(minutes=timestep) leadtimes = [(i + 1) * timestep for i in range(n_timesteps)] timestamps = [startdate + (i + 1) * td for i in range(n_timesteps)] assert (metadata_netcdf["leadtimes"] == leadtimes).all(), "Wrong leadtimes" assert (metadata_netcdf["timestamps"] == timestamps).all(), "Wrong timestamps"
def forecast( precip, precip_metadata, velocity, timesteps, timestep, nowcast_method, precip_nwp=None, precip_nwp_metadata=None, start_blending=120, end_blending=240, fill_nwp=True, nowcast_kwargs=None, ): """Generate a forecast by linearly blending nowcasts with NWP data Parameters ---------- precip: array_like Array containing the input precipitation field(s) ordered by timestamp from oldest to newest. The time steps between the inputs are assumed to be regular. precip_metadata: dict Metadata dictionary containing (at least) the transform, unit and threshold attributes as described in the documentation of :py:mod:`pysteps.io.importers`. velocity; array_like Array of shape (2, m, n) containing the x- and y-components of the advection field. The velocities are assumed to represent one time step between the inputs. All values are required to be finite. timesteps: int Number of time steps to forecast. timestep: int or float The time difference (in minutes) between consecutive forecast fields. nowcast_method: str Name of the nowcasting method. See :py:mod:`pysteps.nowcasts.interface` for the list of available methods. precip_nwp: array_like or NoneType, optional Array of shape (timesteps, m, n) in the case of no ensemble or of shape (n_ens_members, timesteps, m, n) in the case of an ensemble containing the NWP precipitation fields ordered by timestamp from oldest to newest. The time steps between the inputs are assumed to be regular (and identical to the time step between the nowcasts). If no NWP data is given the value of precip_nwp is None and no blending will be performed. precip_nwp_metadata: dict or NoneType, optional NWP metadata dictionary containing (at least) the transform, unit and threshold attributes as described in the documentation of :py:mod:`pysteps.io.importers`. start_blending: int, optional Time stamp (in minutes) after which the blending should start. Before this only the nowcast data is used. end_blending: int, optional Time stamp (in minutes) after which the blending should end. Between start_blending and end_blending the nowcasts and NWP data are linearly merged with each other. After end_blending only the NWP data is used. fill_nwp: bool, optional Standard value is True. If True, the NWP data will be used to fill in the no data mask of the nowcast. nowcast_kwargs: dict, optional Dictionary containing keyword arguments for the nowcast method. Returns ------- R_blended: ndarray Array of shape (timesteps, m, n) in the case of no ensemble or of shape (n_ens_members, timesteps, m, n) in the case of an ensemble containing the precipation forecast generated by linearly blending the nowcasts and the NWP data. n_ens_members equals the maximum no. of ensemble members in either the nowcast or nwp model(s). """ if nowcast_kwargs is None: nowcast_kwargs = dict() # Calculate the nowcasts nowcast_method_func = nowcasts.get_method(nowcast_method) precip_nowcast = nowcast_method_func( precip, velocity, timesteps, **nowcast_kwargs, ) # Make sure that precip_nowcast and precip_nwp are in mm/h precip_nowcast, _ = conversion.to_rainrate(precip_nowcast, metadata=precip_metadata) # Check if NWP data is given as input if precip_nwp is not None: precip_nwp, _ = conversion.to_rainrate(precip_nwp, metadata=precip_nwp_metadata) if len(precip_nowcast.shape) == 4: n_ens_members_nowcast = precip_nowcast.shape[0] if n_ens_members_nowcast == 1: precip_nowcast = np.squeeze(precip_nowcast) else: n_ens_members_nowcast = 1 if len(precip_nwp.shape) == 4: n_ens_members_nwp = precip_nwp.shape[0] if n_ens_members_nwp == 1: precip_nwp = np.squeeze(precip_nwp) else: n_ens_members_nwp = 1 # Now, repeat the nowcast ensemble members or the nwp models/members until # it has the same amount of members as n_ens_members_max. For instance, if # you have 10 ensemble nowcasts members and 3 NWP members, the output will # be an ensemble of 10 members. Hence, the three NWP members are blended # with the first three members of the nowcast (member one with member one, # two with two, etc.), subsequently, the same NWP members are blended with # the next three members (NWP member one with member 4, NWP member 2 with # member 5, etc.), until 10 is reached. n_ens_members_max = max(n_ens_members_nowcast, n_ens_members_nwp) n_ens_members_min = min(n_ens_members_nowcast, n_ens_members_nwp) if n_ens_members_min != n_ens_members_max: if n_ens_members_nwp == 1: precip_nwp = np.repeat( precip_nwp[np.newaxis, :, :], n_ens_members_max, axis=0 ) elif n_ens_members_nowcast == 1: precip_nowcast = np.repeat( precip_nowcast[np.newaxis, :, :], n_ens_members_max, axis=0 ) else: repeats = [ (n_ens_members_max + i) // n_ens_members_min for i in range(n_ens_members_min) ] if n_ens_members_nwp == n_ens_members_min: precip_nwp = np.repeat(precip_nwp, repeats, axis=0) elif n_ens_members_nowcast == n_ens_members_min: precip_nowcast = np.repeat(precip_nowcast, repeats, axis=0) # Check if dimensions are correct assert ( precip_nwp.shape == precip_nowcast.shape ), "The dimensions of precip_nowcast and precip_nwp need to be identical: dimension of precip_nwp = {} and dimension of precip_nowcast = {}".format( precip_nwp.shape, precip_nowcast.shape ) # Initialise output R_blended = np.zeros_like(precip_nowcast) # Calculate the weights for i in range(timesteps): # Calculate what time we are at t = (i + 1) * timestep # Calculate the weight with a linear relation (weight_nwp at start_blending = 0.0) # and (weight_nwp at end_blending = 1.0) weight_nwp = (t - start_blending) / (end_blending - start_blending) # Set weights at times before start_blending and after end_blending if weight_nwp < 0.0: weight_nwp = 0.0 elif weight_nwp > 1.0: weight_nwp = 1.0 # Calculate weight_nowcast weight_nowcast = 1.0 - weight_nwp # Calculate output by combining precip_nwp and precip_nowcast, # while distinguishing between ensemble and non-ensemble methods if n_ens_members_max == 1: R_blended[i, :, :] = ( weight_nwp * precip_nwp[i, :, :] + weight_nowcast * precip_nowcast[i, :, :] ) else: R_blended[:, i, :, :] = ( weight_nwp * precip_nwp[:, i, :, :] + weight_nowcast * precip_nowcast[:, i, :, :] ) # Find where the NaN values are and replace them with NWP data if fill_nwp: nan_indices = np.isnan(R_blended) R_blended[nan_indices] = precip_nwp[nan_indices] else: # If no NWP data is given, the blended field is simply equal to the nowcast field R_blended = precip_nowcast return R_blended
if not np.any(np.isfinite(R[i, :, :])): print("Skipping, no finite values found for time step %d" % (i+1)) missing_data = True break if missing_data: curdate += timedelta(minutes=datasource["timestep"]) continue R[~np.isfinite(R)] = metadata["zerovalue"] R = transformation.dB_transform(R, metadata=metadata)[0] oflow = motion.get_method("lucaskanade") V = oflow(R[-2:, :, :]) nc = nowcasts.get_method("steps") for es in ensemble_sizes: for nw in range(1, max_num_threads+1, 1): starttime = time.time() _, init_time, mainloop_time = \ nc(R[-3:, :, :], V, num_timesteps, n_ens_members=es, n_cascade_levels=6, R_thr=R_min_dB, kmperpixel=1.0, timestep=datasource["timestep"], vel_pert_method="bps", mask_method="incremental", probmatching_method="cdf", num_workers=nw, fft_method=fft_method, measure_time=True) results[es][nw] = (init_time, mainloop_time) # Save the intermediate results for testing purposes. with open("parallel_scaling_results_%s.dat" % domain, "wb") as f: pickle.dump(dict(results), f)
def compute(nowcast_method, config_number): # the optical flow methods to use oflow_methods = ["darts"] # time step between computation of each nowcast (minutes) timestep = 30 # the number of time steps for each nowcast (5 minutes for the MeteoSwiss and # FMI data) num_timesteps = 24 # the threshold to use for precipitation/no precipitation R_min = 0.1 R_min_dB = transformation.dB_transform(np.array([R_min]))[0][0] precip_events = [ ("201104160800", "201104170000"), ("201111152300", "201111161000"), ("201304110000", "201304120000"), ("201304041800", "201304051800"), ("201305180600", "201305181200"), ("201305270000", "201305271200"), ] for pei, pe in enumerate(precip_events): start_date = datetime.strptime(pe[0], "%Y%m%d%H%M") curdate = datetime.strptime(pe[0], "%Y%m%d%H%M") enddate = datetime.strptime(pe[1], "%Y%m%d%H%M") results = {} for m in oflow_methods: results[m] = {} results[m]["comptimes"] = 0.0 results[m]["CSI"] = [0.0] * num_timesteps results[m]["RMSE"] = [0.0] * num_timesteps results[m]["n_samples"] = [0.0] * num_timesteps my_observations = LowAltCompositeCollection() while curdate <= enddate: print("Computing nowcasts for event %d, start date %s..." % (pei + 1, str(curdate)), end="") sys.stdout.flush() if curdate + num_timesteps * timedelta(minutes=5) > enddate: break time_step_in_sec = 5 * 60 times = [ curdate - timedelta(seconds=time_step_in_sec * i) for i in range(9) ] times += [ curdate + timedelta(seconds=time_step_in_sec * i) for i in range(1, num_timesteps + 1) ] times.sort() # Add elements to the collection if they don't exists for _time in times: my_observations.add(get_lowalt_file(_time)) # First 9 times R = my_observations.get_data('Reflectivity', date=times[:9]) R = dbz_to_r(R, a=300., b=1.5) _R = list() # The original data is at 1km resolutions # Downscale to 5 km resolution by 5x5 averaging for i in range(9): _R.append(downscale_local_mean(R[i, :-1, :], (5, 5))) R = np.asarray(_R) my_observations.clean_buffers() # release memory missing_data = False for i in range(R.shape[0]): if not np.any(np.isfinite(R[i, :, :])): print("Skipping, no finite values found for time step %d" % (i + 1)) missing_data = True break if missing_data: curdate += timedelta(minutes=timestep) continue R = transformation.dB_transform(R)[0] # Forecast times fcts_times = times[9:] R_obs = my_observations.get_data('Reflectivity', date=times[9:]) R_obs = dbz_to_r(R_obs, a=300., b=1.5) # The original data is at 1km resolutions # Downscale to 5 km resolution by 5x5 averaging _R = list() for i in range(len(fcts_times)): _R.append(downscale_local_mean(R_obs[i, :-1, :], (5, 5))) R_obs = np.asarray(_R) my_observations.clean_buffers() # release memory for oflow_method in oflow_methods: oflow = motion.get_method(oflow_method) if oflow_method == "vet": R_ = R[-2:, :, :] else: R_ = R starttime = time.time() V = oflow(R_, **configurations[config_number]) print("%s optical flow computed in %.3f seconds." % \ (oflow_method, time.time() - starttime)) if nowcast_method == "advection": nc = nowcasts.get_method("extrapolation") R_fct = nc(R[-1, :, :], V, num_timesteps) else: nc = nowcasts.get_method("steps") R_fct = nc(R[-3:, :, :], V, num_timesteps, noise_method=None, vel_pert_method=None, n_ens_members=1, mask_method="sprog", R_thr=R_min_dB, probmatching_method="mean", fft_method="numpy")[0, :, :, :] R_fct = transformation.dB_transform(R_fct, inverse=True)[0] for lt in range(num_timesteps): if not np.any(np.isfinite(R_obs[lt, :, :])): print( "Warning: no finite verifying observations for lead time %d." % (lt + 1)) continue csi = det_cat_fcst(R_fct[lt, :, :], R_obs[lt, :, :], R_min, ["CSI"])[0] MASK = np.logical_and(R_fct[lt, :, :] > R_min, R_obs[lt, :, :] > R_min) if np.sum(MASK) == 0: print("Skipping, no precipitation for lead time %d." % (lt + 1)) continue rmse = det_cont_fcst(R_fct[lt, :, :][MASK], R_obs[lt, :, :][MASK], ["MAE_add"])[0] results[oflow_method]["CSI"][lt] += csi results[oflow_method]["RMSE"][lt] += rmse results[oflow_method]["n_samples"][lt] += 1 print("Done.") curdate += timedelta(minutes=timestep) data_dir = './data/dart_tests/config_{:d}'.format(config_number) create_dir(data_dir) file_name = "optflow_comparison_results_%s_%s.dat" % ( nowcast_method, get_timestamp(start_date)) with open(join(data_dir, file_name), "wb") as f: pickle.dump(results, f)
continue for oflow_method in oflow_methods: oflow = motion.get_method(oflow_method) if oflow_method == "vet": R_ = R[-2:, :, :] else: R_ = R starttime = time.time() V = oflow(R_) print("%s optical flow computed in %.3f seconds." % \ (oflow_method, time.time() - starttime)) if nowcast_method == "advection": nc = nowcasts.get_method("extrapolation") R_fct = nc(R[-1, :, :], V, num_timesteps) else: nc = nowcasts.get_method("steps") R_fct = nc(R[-3:, :, :], V, num_timesteps, noise_method=None, vel_pert_method=None, n_ens_members=1, mask_method="sprog", R_thr=R_min_dB, probmatching_method="mean", fft_method="numpy")[0, :, :, :] R_fct = transformation.dB_transform(R_fct, inverse=True)[0]