def import_mch_gif(): date = datetime.datetime.strptime("201505151630", "%Y%m%d%H%M") data_source = "mch" # Load data source config root_path = rcparams.data_sources[data_source]["root_path"] path_fmt = rcparams.data_sources[data_source]["path_fmt"] fn_pattern = rcparams.data_sources[data_source]["fn_pattern"] fn_ext = rcparams.data_sources[data_source]["fn_ext"] importer_name = rcparams.data_sources[data_source]["importer"] importer_kwargs = rcparams.data_sources[data_source]["importer_kwargs"] timestep = rcparams.data_sources[data_source]["timestep"] # Find the input files from the archive fns = io.archive.find_by_date(date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_prev_files=1) # Read the radar composites importer = io.get_method(importer_name, "importer") R, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs) return R, metadata
def _import_mch_gif(prv, nxt): date = datetime.datetime.strptime("201505151630", "%Y%m%d%H%M") data_source = rcparams.data_sources["mch"] # Load data source config root_path = data_source["root_path"] path_fmt = data_source["path_fmt"] fn_pattern = data_source["fn_pattern"] fn_ext = data_source["fn_ext"] importer_name = data_source["importer"] importer_kwargs = data_source["importer_kwargs"] timestep = data_source["timestep"] # Find the input files from the archive fns = io.archive.find_by_date( date, root_path, path_fmt, fn_pattern, fn_ext, timestep=timestep, num_prev_files=prv, num_next_files=nxt, ) # Read the radar composites importer = io.get_method(importer_name, "importer") R, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs) # Convert to rain rate R, metadata = utils.conversion.to_rainrate(R, metadata) # Upscale data to 2 km R, metadata = utils.dimension.aggregate_fields_space(R, metadata, 2000) # Log-transform the data to unit of dBR, set the threshold to 0.1 mm/h, # set the fill value to -15 dBR R, metadata = utils.transformation.dB_transform(R, metadata, threshold=0.1, zerovalue=-15.0) # Set missing values with the fill value R[~np.isfinite(R)] = -15.0 return R, metadata
def get_precipitation_fields(num_prev_files=0): """Get a precipitation field from the archive to be used as reference.""" # Selected case date = datetime.strptime("201505151630", "%Y%m%d%H%M") data_source = rcparams.data_sources["mch"] root_path = data_source["root_path"] path_fmt = data_source["path_fmt"] fn_pattern = data_source["fn_pattern"] fn_ext = data_source["fn_ext"] importer_name = data_source["importer"] importer_kwargs = data_source["importer_kwargs"] # Find the input files from the archive fns = io.archive.find_by_date(date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_prev_files=num_prev_files) # Read the radar composites importer = io.get_method(importer_name, "importer") reference_field, quality, metadata = io.read_timeseries( fns, importer, **importer_kwargs) del quality # Not used if num_prev_files == 0: reference_field = np.squeeze(reference_field) # Remove time dimension # Convert to mm/h reference_field, metadata = stp.utils.to_rainrate(reference_field, metadata) # Mask invalid values reference_field = np.ma.masked_invalid(reference_field) # Log-transform the data [dBR] reference_field, metadata = stp.utils.dB_transform(reference_field, metadata, threshold=0.1, zerovalue=-15.0) return reference_field
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ root_path = data_source["root_path"] path_fmt = data_source["path_fmt"] fn_pattern = data_source["fn_pattern"] fn_ext = data_source["fn_ext"] importer_name = data_source["importer"] importer_kwargs = data_source["importer_kwargs"] # Find the reference field in the archive fns = io.archive.find_by_date(date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_prev_files=0) # Read the reference radar composite importer = io.get_method(importer_name, "importer") reference_field, quality, metadata = io.read_timeseries(fns, importer, **importer_kwargs) del quality # Not used reference_field = np.squeeze(reference_field) # Remove time dimension ############################################################################### # Preprocess the data # ~~~~~~~~~~~~~~~~~~~ # Convert to mm/h reference_field, metadata = stp.utils.to_rainrate(reference_field, metadata) # Mask invalid values reference_field = np.ma.masked_invalid(reference_field)
from pysteps.io.importers import import_fmi_pgm from pysteps.cascade.decomposition import decomposition_fft date = datetime.strptime("201609281600", "%Y%m%d%H%M") # insert your data path here root_path = "" fn_pattern = "%Y%m%d%H%M_fmi.radar.composite.lowest_FIN_SUOMI1" fn_ext = "pgm.gz" cmap = cm.RdBu_r vmin = -3 vmax = 3 fn = io.archive.find_by_date(date, root_path, "%Y%m%d", fn_pattern, fn_ext, 5) R, _, metadata = io.read_timeseries(fn, import_fmi_pgm, gzipped=True) R = R.squeeze() R[R < 10.0] = 5.0 R[~np.isfinite(R)] = 5.0 R = (R - np.mean(R)) / np.std(R) filter = filter_gaussian(R.shape, 8) decomp = decomposition_fft(R, filter) for i in range(8): mu = decomp["means"][i] sigma = decomp["stds"][i] decomp["cascade_levels"][i] = (decomp["cascade_levels"][i] - mu) / sigma fig, ax = pyplot.subplots(nrows=2, ncols=4)
source = "".join([i for i in case if not i.isdigit()]) data_source = pysteps.rcparams.data_sources[source] # Find the input files from the archive file_names = io.archive.find_by_date( case_date, data_source["root_path"], data_source["path_fmt"], data_source["fn_pattern"], data_source["fn_ext"], data_source["timestep"], num_prev_files=0, num_next_files=frames - 1, ) if None in file_names[0]: raise FileNotFoundError( f"Error loading {case} case. Some files are missing.") # Read the radar composites importer = io.get_method(data_source["importer"], "importer") importer_kwargs = data_source["importer_kwargs"] reflectivity, _, metadata = io.read_timeseries(file_names, importer, **importer_kwargs) # Convert to rain rate precip, metadata = conversion.to_rainrate(reflectivity, metadata) return precip, metadata, data_source["timestep"]
importer_name = data_source["importer"] importer_kwargs = data_source["importer_kwargs"] timestep = data_source["timestep"] # Find the two input files from the archive fns = io.archive.find_by_date(date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_prev_files=1) # Read the radar composites importer = io.get_method(importer_name, "importer") R, quality, metadata = io.read_timeseries(fns, importer, **importer_kwargs) del quality # Not used ############################################################################### # Preprocess the data # ~~~~~~~~~~~~~~~~~~~ # Convert to mm/h R, metadata = conversion.to_rainrate(R, metadata) # Keep the reference frame in mm/h and its mask (for plotting purposes) ref_mm = R[0, :, :].copy() mask = np.ones(ref_mm.shape) mask[~np.isnan(ref_mm)] = np.nan
fn_pattern = data_source["fn_pattern"] fn_ext = data_source["fn_ext"] importer_name = data_source["importer"] importer_kwargs = data_source["importer_kwargs"] timestep = data_source["timestep"] # Load the data from the archive fns = io.archive.find_by_date(date, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_next_files=20) importer = io.get_method(importer_name, "importer") R, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs) # Convert to reflectivity (it is possible to give the a- and b- parameters of the # Marshall-Palmer relationship here: zr_a = and zr_b =). Z, metadata = to_reflectivity(R, metadata) # Extract the list of timestamps timelist = metadata["timestamps"] pprint(metadata) ############################################################################### # Example of thunderstorm identification in a single timestep. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # The function tstorm_detect.detection requires a 2-D input image, all further inputs are # optional.
def get_precipitation_fields( num_prev_files=0, num_next_files=0, return_raw=False, metadata=False, upscale=None, source="mch", log_transform=True, clip=None, **importer_kwargs, ): """ Get a precipitation field from the archive to be used as reference. Source: bom Reference time: 2018/06/16 10000 UTC Source: fmi Reference time: 2016/09/28 1600 UTC Source: knmi Reference time: 2010/08/26 0000 UTC Source: mch Reference time: 2015/05/15 1630 UTC Source: opera Reference time: 2018/08/24 1800 UTC Source: saf Reference time: 2018/06/01 0700 UTC Source: mrms Reference time: 2019/06/10 0000 UTC Parameters ---------- num_prev_files: int, optional Number of previous times (files) to return with respect to the reference time. num_next_files: int, optional Number of future times (files) to return with respect to the reference time. return_raw: bool, optional Do not preprocess the precipitation fields. False by default. The pre-processing steps are: 1) Convert to mm/h, 2) Mask invalid values, 3) Log-transform the data [dBR]. metadata: bool, optional If True, also return file metadata. clip: scalars (left, right, bottom, top), optional The extent of the bounding box in data coordinates to be used to clip the data. upscale: float or None, optional Upscale fields in space during the pre-processing steps. If it is None, the precipitation field is not modified. If it is a float, represents the length of the space window that is used to upscale the fields. source: {"bom", "fmi" , "knmi", "mch", "opera", "saf", "mrms"}, optional Name of the data source to be used. log_transform: bool Whether to transform the output to dB. Other Parameters ---------------- importer_kwargs : dict Additional keyword arguments passed to the importer. Returns ------- reference_field : array metadata : dict """ if source == "bom": pytest.importorskip("netCDF4") if source == "fmi": pytest.importorskip("pyproj") if source == "knmi": pytest.importorskip("h5py") if source == "mch": pytest.importorskip("PIL") if source == "opera": pytest.importorskip("h5py") if source == "saf": pytest.importorskip("netCDF4") if source == "mrms": pytest.importorskip("pygrib") try: date = _reference_dates[source] except KeyError: raise ValueError( f"Unknown source name '{source}'\n" "The available data sources are: " f"{str(list(_reference_dates.keys()))}" ) data_source = rcparams.data_sources[source] root_path = data_source["root_path"] path_fmt = data_source["path_fmt"] fn_pattern = data_source["fn_pattern"] fn_ext = data_source["fn_ext"] importer_name = data_source["importer"] _importer_kwargs = data_source["importer_kwargs"].copy() _importer_kwargs.update(**importer_kwargs) timestep = data_source["timestep"] # Find the input files from the archive fns = io.archive.find_by_date( date, root_path, path_fmt, fn_pattern, fn_ext, timestep=timestep, num_prev_files=num_prev_files, num_next_files=num_next_files, ) # Read the radar composites importer = io.get_method(importer_name, "importer") reference_field, __, ref_metadata = io.read_timeseries( fns, importer, **_importer_kwargs ) if not return_raw: if (num_prev_files == 0) and (num_next_files == 0): # Remove time dimension reference_field = np.squeeze(reference_field) # Convert to mm/h reference_field, ref_metadata = stp.utils.to_rainrate( reference_field, ref_metadata ) # Clip domain reference_field, ref_metadata = stp.utils.clip_domain( reference_field, ref_metadata, clip ) # Upscale data reference_field, ref_metadata = aggregate_fields_space( reference_field, ref_metadata, upscale ) # Mask invalid values reference_field = np.ma.masked_invalid(reference_field) if log_transform: # Log-transform the data [dBR] reference_field, ref_metadata = stp.utils.dB_transform( reference_field, ref_metadata, threshold=0.1, zerovalue=-15.0 ) # Set missing values with the fill value np.ma.set_fill_value(reference_field, ref_metadata["zerovalue"]) reference_field.data[reference_field.mask] = ref_metadata["zerovalue"] if metadata: return reference_field, ref_metadata return reference_field
ext = rcparams.data_sources["mch"]["fn_ext"] timestep = rcparams.data_sources["mch"]["timestep"] importer_name = rcparams.data_sources["mch"]["importer"] importer_kwargs = rcparams.data_sources["mch"]["importer_kwargs"] # read precip field date = datetime.strptime("201607112100", "%Y%m%d%H%M") fns = io.find_by_date(date, root, fmt, pattern, ext, timestep, num_prev_files=2) importer = io.get_method(importer_name, "importer") precip, __, metadata = io.read_timeseries(fns, importer, **importer_kwargs) precip, metadata = utils.to_rainrate(precip, metadata) # precip[np.isnan(precip)] = 0 # motion motion = dense_lucaskanade(precip) # parameters nleadtimes = 6 thr = 1 # mm / h slope = 1 * timestep # km / min # compute probability forecast extrap_kwargs = dict(allow_nonfinite_values=True) fct = forecast(precip[-1], motion,
root_path = radar_data_source["root_path"] path_fmt = "prcp-c10/66/%Y/%m/%d" fn_pattern = "66_%Y%m%d_%H%M00.prcp-c10" fn_ext = radar_data_source["fn_ext"] importer_name = radar_data_source["importer"] importer_kwargs = radar_data_source["importer_kwargs"] timestep = 10.0 # Find the radar files in the archive fns = io.find_by_date( date_radar, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_prev_files=2 ) # Read the radar composites importer = io.get_method(importer_name, "importer") radar_precip, _, radar_metadata = io.read_timeseries(fns, importer, **importer_kwargs) # Import the NWP data filename = os.path.join( nwp_data_source["root_path"], datetime.strftime(date_nwp, nwp_data_source["path_fmt"]), datetime.strftime(date_nwp, nwp_data_source["fn_pattern"]) + "." + nwp_data_source["fn_ext"], ) nwp_importer = io.get_method("bom_nwp", "importer") nwp_precip, _, nwp_metadata = nwp_importer(filename) # Only keep the NWP forecasts from the last radar observation time (2020-10-31 04:00) # onwards
datasource_params = rcparams.data_sources[data_source] # Find the radar files in the archive fns = io.find_by_date( date, datasource_params["root_path"], datasource_params["path_fmt"], datasource_params["fn_pattern"], datasource_params["fn_ext"], datasource_params["timestep"], num_prev_files=2, ) # Read the data from the archive importer = io.get_method(datasource_params["importer"], "importer") reflectivity, _, metadata = io.read_timeseries( fns, importer, **datasource_params["importer_kwargs"]) # Convert reflectivity to rain rate rainrate, metadata = conversion.to_rainrate(reflectivity, metadata) # Upscale data to 2 km to reduce computation time rainrate, metadata = dimension.aggregate_fields_space(rainrate, metadata, 2000) # Plot the most recent rain rate field plt.figure() plot_precip_field(rainrate[-1, :, :]) plt.show() ############################################################################### # Estimate the advection field # ----------------------------
rho.pop(0) if np.isfinite(rho[1]): results["cc_ar"][i][t] += rho[1] results["n_ar_samples"][i][t] += 1 k += 1 R_ep = extrapolator(R[-1, :, :], V, num_timesteps, outval=R_min) obs_fns = io.archive.find_by_date(curdate, root_path, datasource["path_fmt"], datasource["fn_pattern"], datasource["fn_ext"], datasource["timestep"], num_next_files=num_timesteps) R_obs, _, metadata = io.read_timeseries( obs_fns, importer, **datasource["importer_kwargs"]) R_obs = R_obs[1:, :, :] R_obs, metadata = utils.conversion.to_rainrate(R_obs, metadata, a=to_rainrate_a, b=to_rainrate_b) R_obs, metadata = utils.transformation.dB_transform(R_obs, metadata, threshold=R_thr) R_min = np.min(R_obs[np.isfinite(R_obs)]) R_obs[~np.isfinite(R_obs)] = R_min for t in range(num_timesteps): if not np.any(np.isfinite(R_obs[t, :, :])): print(
root_path = data_source["root_path"] path_fmt = data_source["path_fmt"] fn_pattern = data_source["fn_pattern"] fn_ext = data_source["fn_ext"] importer_name = data_source["importer"] importer_kwargs = data_source["importer_kwargs"] # Find the input files in the archive. Use history length of 5 timesteps filenames = io.archive.find_by_date( date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_prev_files=5 ) # Read the input time series importer = io.get_method(importer_name, "importer") rainrate_field, quality, metadata = io.read_timeseries( filenames, importer, **importer_kwargs ) # Convert to rain rate (mm/h) rainrate_field, metadata = utils.to_rainrate(rainrate_field, metadata) ################################################################################ # Compute the advection field # --------------------------- # # Apply the Lucas-Kanade method with the parameters given in Pulkkinen et al. # (2020) to compute the advection field. fd_kwargs = {} fd_kwargs["max_corners"] = 1000 fd_kwargs["quality_level"] = 0.01
def get_precipitation_fields(num_prev_files=0, num_next_files=0, return_raw=False, metadata=False, upscale=None): """ Get a precipitation field from the archive to be used as reference. Source: mch Reference time: 2015/05/15 1630 UTC Parameters ---------- num_prev_files: int Number of previous times (files) to return with respect to the reference time. num_next_files: int Number of future times (files) to return with respect to the reference time. return_raw: bool Do not preprocess the precipitation fields. False by default. The pre-processing steps are: 1) Convert to mm/h, 2) Mask invalid values, 3) Log-transform the data [dBR]. metadata : bool If True, also return file metadata. upscale: float or None Upscale fields in space during the pre-processing steps. If it is None, the precipitation field is not modified. If it is a float, represents the length of the space window that is used to upscale the fields. Returns ------- reference_field : array metadata : dict """ pytest.importorskip('PIL') # Selected case date = datetime.strptime("201505151630", "%Y%m%d%H%M") data_source = rcparams.data_sources["mch"] root_path = data_source["root_path"] path_fmt = data_source["path_fmt"] fn_pattern = data_source["fn_pattern"] fn_ext = data_source["fn_ext"] importer_name = data_source["importer"] importer_kwargs = data_source["importer_kwargs"] timestep = data_source["timestep"] # Find the input files from the archive fns = io.archive.find_by_date(date, root_path, path_fmt, fn_pattern, fn_ext, timestep=timestep, num_prev_files=num_prev_files, num_next_files=num_next_files) # Read the radar composites importer = io.get_method(importer_name, "importer") reference_field, __, ref_metadata = io.read_timeseries(fns, importer, **importer_kwargs) if not return_raw: if (num_prev_files == 0) and (num_next_files == 0): # Remove time dimension reference_field = np.squeeze(reference_field) # Convert to mm/h reference_field, ref_metadata = stp.utils.to_rainrate(reference_field, ref_metadata) # Upscale data to 2 km reference_field, ref_metadata = aggregate_fields_space(reference_field, ref_metadata, upscale) # Mask invalid values reference_field = np.ma.masked_invalid(reference_field) # Log-transform the data [dBR] reference_field, ref_metadata = stp.utils.dB_transform(reference_field, ref_metadata, threshold=0.1, zerovalue=-15.0) # Set missing values with the fill value reference_field.data[reference_field.mask] = -15.0 if metadata: return reference_field, ref_metadata return reference_field