def test_dB_transform(R, metadata, threshold, zerovalue, inverse, expected): """Test the dB_transform.""" assert_array_almost_equal( transformation.dB_transform(R, metadata, threshold, zerovalue, inverse)[0], expected, )
# Compute 30-minute LINDA nowcast with 8 parallel workers # Restrict the number of features to 15 to reduce computation time nowcast_linda = linda.forecast( rainrate, advection, 6, max_num_features=15, add_perturbations=False, num_workers=8, measure_time=True, )[0] # Compute S-PROG nowcast for comparison rainrate_db, _ = transformation.dB_transform(rainrate, metadata, threshold=0.1, zerovalue=-15.0) nowcast_sprog = sprog.forecast( rainrate_db[-3:, :, :], advection, 6, n_cascade_levels=6, R_thr=-10.0, ) # Convert reflectivity nowcast to rain rate nowcast_sprog = transformation.dB_transform(nowcast_sprog, threshold=-10.0, inverse=True)[0] # Plot the nowcasts
else: datasource = datasources.mch precipevents = precipevents.mch root_path = os.path.join(datasources.root_path, datasource["root_path"]) importer = io.get_method(datasource["importer"], "importer") results = {} for m in oflow_methods: results[m] = {} results[m]["CSI"] = [0.0] * num_timesteps results[m]["MAE"] = [0.0] * num_timesteps results[m]["n_samples"] = [0.0] * num_timesteps R_min_dB = transformation.dB_transform(np.array([R_min]))[0][0] for pei, pe in enumerate(precipevents): curdate = datetime.strptime(pe[0], "%Y%m%d%H%M") enddate = datetime.strptime(pe[1], "%Y%m%d%H%M") while curdate <= enddate: print("Computing nowcasts for event %d, start date %s..." % (pei + 1, str(curdate)), end="") sys.stdout.flush() if curdate + num_timesteps * timedelta(minutes=5) > enddate: break fns = io.archive.find_by_date(curdate,
rainrate_field_log, _ = utils.transformation.dB_transform( rainrate_field, metadata=metadata ) velocity = oflow(rainrate_field_log, **oflow_kwargs) ############################################################################### # Compute the nowcasts and threshold rain rates below 0.5 mm/h # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ forecast_extrap = extrapolation.forecast( rainrate_field[-1], velocity, 3, extrap_kwargs={"allow_nonfinite_values": True} ) forecast_extrap[forecast_extrap < 0.5] = 0.0 # log-transform the data and the threshold value to dBR units for S-PROG rainrate_field_db, _ = transformation.dB_transform( rainrate_field, metadata, threshold=0.1, zerovalue=-15.0 ) rainrate_thr, _ = transformation.dB_transform( np.array([0.5]), metadata, threshold=0.1, zerovalue=-15.0 ) forecast_sprog = sprog.forecast( rainrate_field_db[-3:], velocity, 3, n_cascade_levels=8, R_thr=rainrate_thr[0] ) forecast_sprog, _ = transformation.dB_transform( forecast_sprog, threshold=-10.0, inverse=True ) forecast_sprog[forecast_sprog < 0.5] = 0.0 forecast_anvil = anvil.forecast( rainrate_field[-4:], velocity, 3, ar_window_radius=25, ar_order=2 )
vsf = 60.0 / datasource["timestep"] * metadata["xpixelsize"] / 1000.0 missing_data = False for i in range(R.shape[0]): if not np.any(np.isfinite(R[i, :, :])): missing_data = True break if missing_data: curdate += timedelta(minutes=datasource["timestep"]) continue R[~np.isfinite(R)] = metadata["zerovalue"] if use_precip_mask: MASK = np.any(R < R_min, axis=0) R = transformation.dB_transform(R)[0] if args.oflow == "vet": R_ = R[-2:, :, :] else: R_ = R # TODO: Allow the user to supply parameters for the optical flow. V = oflow(R_) * vsf # discard the motion field if the mean velocity is abnormally large if np.nanmean(np.linalg.norm(V, axis=0)) > 0.5 * R.shape[1]: curdate += timedelta(minutes=datasource["timestep"]) continue if use_precip_mask: V[0, :, :][MASK] = np.nan
def compute(nowcast_method, config_number): # the optical flow methods to use oflow_methods = ["darts"] # time step between computation of each nowcast (minutes) timestep = 30 # the number of time steps for each nowcast (5 minutes for the MeteoSwiss and # FMI data) num_timesteps = 24 # the threshold to use for precipitation/no precipitation R_min = 0.1 R_min_dB = transformation.dB_transform(np.array([R_min]))[0][0] precip_events = [ ("201104160800", "201104170000"), ("201111152300", "201111161000"), ("201304110000", "201304120000"), ("201304041800", "201304051800"), ("201305180600", "201305181200"), ("201305270000", "201305271200"), ] for pei, pe in enumerate(precip_events): start_date = datetime.strptime(pe[0], "%Y%m%d%H%M") curdate = datetime.strptime(pe[0], "%Y%m%d%H%M") enddate = datetime.strptime(pe[1], "%Y%m%d%H%M") results = {} for m in oflow_methods: results[m] = {} results[m]["comptimes"] = 0.0 results[m]["CSI"] = [0.0] * num_timesteps results[m]["RMSE"] = [0.0] * num_timesteps results[m]["n_samples"] = [0.0] * num_timesteps my_observations = LowAltCompositeCollection() while curdate <= enddate: print("Computing nowcasts for event %d, start date %s..." % (pei + 1, str(curdate)), end="") sys.stdout.flush() if curdate + num_timesteps * timedelta(minutes=5) > enddate: break time_step_in_sec = 5 * 60 times = [ curdate - timedelta(seconds=time_step_in_sec * i) for i in range(9) ] times += [ curdate + timedelta(seconds=time_step_in_sec * i) for i in range(1, num_timesteps + 1) ] times.sort() # Add elements to the collection if they don't exists for _time in times: my_observations.add(get_lowalt_file(_time)) # First 9 times R = my_observations.get_data('Reflectivity', date=times[:9]) R = dbz_to_r(R, a=300., b=1.5) _R = list() # The original data is at 1km resolutions # Downscale to 5 km resolution by 5x5 averaging for i in range(9): _R.append(downscale_local_mean(R[i, :-1, :], (5, 5))) R = np.asarray(_R) my_observations.clean_buffers() # release memory missing_data = False for i in range(R.shape[0]): if not np.any(np.isfinite(R[i, :, :])): print("Skipping, no finite values found for time step %d" % (i + 1)) missing_data = True break if missing_data: curdate += timedelta(minutes=timestep) continue R = transformation.dB_transform(R)[0] # Forecast times fcts_times = times[9:] R_obs = my_observations.get_data('Reflectivity', date=times[9:]) R_obs = dbz_to_r(R_obs, a=300., b=1.5) # The original data is at 1km resolutions # Downscale to 5 km resolution by 5x5 averaging _R = list() for i in range(len(fcts_times)): _R.append(downscale_local_mean(R_obs[i, :-1, :], (5, 5))) R_obs = np.asarray(_R) my_observations.clean_buffers() # release memory for oflow_method in oflow_methods: oflow = motion.get_method(oflow_method) if oflow_method == "vet": R_ = R[-2:, :, :] else: R_ = R starttime = time.time() V = oflow(R_, **configurations[config_number]) print("%s optical flow computed in %.3f seconds." % \ (oflow_method, time.time() - starttime)) if nowcast_method == "advection": nc = nowcasts.get_method("extrapolation") R_fct = nc(R[-1, :, :], V, num_timesteps) else: nc = nowcasts.get_method("steps") R_fct = nc(R[-3:, :, :], V, num_timesteps, noise_method=None, vel_pert_method=None, n_ens_members=1, mask_method="sprog", R_thr=R_min_dB, probmatching_method="mean", fft_method="numpy")[0, :, :, :] R_fct = transformation.dB_transform(R_fct, inverse=True)[0] for lt in range(num_timesteps): if not np.any(np.isfinite(R_obs[lt, :, :])): print( "Warning: no finite verifying observations for lead time %d." % (lt + 1)) continue csi = det_cat_fcst(R_fct[lt, :, :], R_obs[lt, :, :], R_min, ["CSI"])[0] MASK = np.logical_and(R_fct[lt, :, :] > R_min, R_obs[lt, :, :] > R_min) if np.sum(MASK) == 0: print("Skipping, no precipitation for lead time %d." % (lt + 1)) continue rmse = det_cont_fcst(R_fct[lt, :, :][MASK], R_obs[lt, :, :][MASK], ["MAE_add"])[0] results[oflow_method]["CSI"][lt] += csi results[oflow_method]["RMSE"][lt] += rmse results[oflow_method]["n_samples"][lt] += 1 print("Done.") curdate += timedelta(minutes=timestep) data_dir = './data/dart_tests/config_{:d}'.format(config_number) create_dir(data_dir) file_name = "optflow_comparison_results_%s_%s.dat" % ( nowcast_method, get_timestamp(start_date)) with open(join(data_dir, file_name), "wb") as f: pickle.dump(results, f)
fn_pattern = "%Y%m%d%H%M_fmi.radar.composite.lowest_FIN_SUOMI1" fn_ext = "pgm.gz" # find the input files from the archive fns = io.archive.find_by_date(date, root_path, "%Y%m%d", fn_pattern, fn_ext, 5, num_prev_files=9) # read the radar composites and apply thresholding Z, _, metadata = io.read_timeseries(fns, import_fmi_pgm, gzipped=True) R = conversion.to_rainrate(Z, metadata, 223.0, 1.53)[0] R = transformation.dB_transform(R, threshold=0.1, zerovalue=-15.0)[0] R[~np.isfinite(R)] = -15.0 # construct bandpass filter and apply the cascade decomposition filter = filter_gaussian(R.shape[1:], 7) decomp = decomposition_fft(R[-1, :, :], filter) # plot the normalized cascade levels for i in range(7): mu = decomp["means"][i] sigma = decomp["stds"][i] decomp["cascade_levels"][i] = (decomp["cascade_levels"][i] - mu) / sigma fig, ax = pyplot.subplots(nrows=2, ncols=4) ax[0, 0].imshow(R[-1, :, :], cmap=cm.RdBu_r, vmin=-3, vmax=3)
seed = 24 map_plotter = "basemap" basemap_resolution = 'h' inputfns = find_by_date(date, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_prev_files=9) Z, _, metadata = read_timeseries(inputfns, import_fmi_pgm, gzipped=True) R = conversion.to_rainrate(Z, metadata, 223.0, 1.53)[0] R = transformation.dB_transform(R, threshold=0.1, zerovalue=-15.0)[0] R[~np.isfinite(R)] = -15.0 V = dense_lucaskanade(R) # the S-PROG nowcast nowcast_method = nowcasts.get_method("sprog") R_f = nowcast_method(R[-3:, :, :], V, 12, n_cascade_levels=8, R_thr=-10.0, decomp_method="fft", bandpass_filter_method="gaussian", probmatching_method="mean", fft_method="pyfftw")