def read_convert_METRANET(filepath, ZDC=None): """This function reads in MeteoSwiss-Metranet-Files and converts values according to data-scale""" met_file = metranet.read_file(filepath) if met_file is None: return None data = met_file.data scale = met_file.scale if ZDC: scale = scale*200 converted = np.apply_along_axis(convert2scale, 1, data, scale) if ZDC: converted[np.where(converted < 600.)] = np.nan converted[np.where(converted > 6000.)] = np.nan return converted
def import_mch_metranet(filename, product, unit, accutime): """Import a 8-bit bin radar reflectivity composite from the MeteoSwiss archive. Parameters ---------- filename : str Name of the file to import. product : {"AQC", "CPC", "RZC", "AZC"} The name of the MeteoSwiss QPE product.\n Currently supported prducts: +------+----------------------------+ | Name | Product | +======+============================+ | AQC | Acquire | +------+----------------------------+ | CPC | CombiPrecip | +------+----------------------------+ | RZC | Precip | +------+----------------------------+ | AZC | RZC accumulation | +------+----------------------------+ unit : {"mm/h", "mm", "dBZ"} the physical unit of the data accutime : float the accumulation time in minutes of the data Returns ------- out : tuple A three-element tuple containing the precipitation field in mm/h imported from a MeteoSwiss gif file and the associated quality field and metadata. The quality field is currently set to None. """ if not metranet_imported: raise MissingOptionalDependency( "metranet package needed for importing MeteoSwiss " "radar composites but it is not installed") ret = metranet.read_file(filename, physic_value=True, verbose=False) R = ret.data geodata = _import_mch_geodata() # read metranet metadata = geodata metadata["institution"] = "MeteoSwiss" metadata["accutime"] = accutime metadata["unit"] = unit metadata["transform"] = None metadata["zerovalue"] = np.nanmin(R) if np.isnan(metadata["zerovalue"]): metadata["threshold"] = np.nan else: metadata["threshold"] = np.nanmin(R[R > metadata["zerovalue"]]) metadata["zr_a"] = 316.0 metadata["zr_b"] = 1.5 return R, None, metadata
import sys sys.path.append('/proj/lom/python/library/radar/io') import metranet radar_file="AZC1830312007L.812" ret = metranet.read_file(radar_file, physic_value=True, verbose=True) ret.data.tofile("AZC1830312007L.812.dat")
def calc_disparr(t_current, cfg_set, resid=False): """Get 2-dim displacement array for flow between timesteps t_current and t_current - n_past_frames*timestep. Parameters ---------- t_current : datetime object Current time for which to calculate displacement array. cfg_set : dict Basic variables defined in input_NOSTRADAMUS_ANN.py resid : bool Do displacement array creation for residual movement correction? Default: False. UV_inter : bool If UV_inter is true, the calculated UV and sparsened UV vectors are returned as well. See function check_create_disparray(t0, timestep, n_integ, root_path, t0_str, oflow_source, oflow_source_path) """ if resid: ## Read in current displaced oflow_source file: filename = pth.path_creator_vararr("disp",cfg_set["oflow_source"],cfg_set) #filename = "%stmp/%s_%s_disp%s.%s" % (cfg_set["root_path"], cfg_set["t0"].strftime("%Y%m%d%H%M"), # cfg_set["oflow_source"], cfg_set["file_ext_verif"], cfg_set["save_type"]) t_diff = cfg_set["t0"] - t_current t_diff_ind = int((t_diff.seconds/60)/cfg_set["timestep"]) #oflow_source_data = np.load(filename)[t_diff_ind:t_diff_ind+cfg_set["n_past_frames"]+1,:,:] #oflow_source_data = np.load(filename)[t_diff_ind+cfg_set["n_past_frames_resid"]::-1,:,:][:cfg_set["n_past_frames"]+1] oflow_source_data = iotmp.load_file(filename,cfg_set["oflow_source"])[t_diff_ind+cfg_set["n_past_frames_resid"]::-1,:,:][:cfg_set["n_past_frames"]+1] if oflow_source_data.shape[0]==1: UV = R = np.zeros((2,oflow_source_data.shape[1],oflow_source_data.shape[2])) if not cfg_set["UV_inter"]: return UV, R else: return UV, R, np.zeros(4)*np.nan, np.zeros(4)*np.nan if np.all(np.array_equal(oflow_source_data[0,:,:],oflow_source_data[1,:,:])): raise ValueError("Input data equal") else: ## Read in current oflow_source file: filenames, timestamps = pth.path_creator(t_current, cfg_set["oflow_source"], cfg_set["source_dict"][cfg_set["oflow_source"]], cfg_set) ret = metranet.read_file(filenames[0], physic_value=True) oflow_source_data = np.atleast_3d(ret.data) for filename in filenames[1:]: ret_d_t = metranet.read_file(filename, physic_value=True) oflow_source_data_d_t = np.atleast_3d(ret_d_t.data) oflow_source_data = np.append(oflow_source_data,oflow_source_data_d_t, axis=2) oflow_source_data = np.moveaxis(oflow_source_data,2,0) #oflow_source_data_masked = np.ma.masked_invalid(oflow_source_data) #oflow_source_data_masked = np.ma.masked_where(oflow_source_data_masked==0,oflow_source_data_masked) ## Check whether there are non-nan entries: if np.any(np.isnan(oflow_source_data).all(axis=(1,2))): print(" *** Warning: Input oflow source field is all NAN!\n Returning NAN fields.***") nan_arr = oflow_source_data[0,:,:]*np.nan D = np.array([nan_arr,nan_arr]) UV = np.array([nan_arr,nan_arr]) UV_vec = []; UV_vec_sp = [] if not cfg_set["UV_inter"]: return D, UV else: return D, UV, UV_vec, UV_vec_sp ## Convert linear rainrates to logarithimc dBR units if not cfg_set["oflow_source"]=="RZC": raise NotImplementedError("So far displacement array retrieval only implemented for RZC") else: ## Get threshold method: if not resid: R_thresh_meth = cfg_set["R_thresh_meth"] R_threshold = cfg_set["R_threshold"] else: R_thresh_meth = cfg_set["R_thresh_meth_resid"] R_threshold = cfg_set["R_threshold_resid"] ## Get threshold value: if R_thresh_meth == "fix": R_thresh = R_threshold elif R_thresh_meth == "perc": R_thresh = np.min([np.nanpercentile(oflow_source_data[0,:,:],R_threshold), np.nanpercentile(oflow_source_data[1,:,:],R_threshold)]) else: raise ValueError("R_thresh_meth must either be set to 'fix' or 'perc'") ## Convert to dBR dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_thresh) dBR[~np.isfinite(dBR)] = dBRmin #R_thresh = cfg_set["R_threshold"] ## In case threshold is not exceeded, lower R_threshold by 20% while (dBR==dBRmin).all(): if cfg_set["verbose"]: print(" *** Warning: Threshold not exceeded, "+ "lower R_threshold by 20% to "+str(R_thresh*0.8)+" ***") R_thresh = R_thresh*0.8 dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_thresh) dBR[~np.isfinite(dBR)] = dBRmin ## For correction of residuals original mm/h values are used: if resid: oflow_source_data_min = oflow_source_data oflow_source_data_min[oflow_source_data_min<=R_thresh] = R_thresh oflow_source_data_min[~np.isfinite(oflow_source_data_min)] = R_thresh ## Calculate UV field oflow_method = st.optflow.get_method(cfg_set["oflow_method_name"]) if not resid: UV, UV_vec, UV_vec_sp = oflow_method(dBR,return_single_vec=True,return_declust_vec=True) else: UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,winsize_LK5=(120,20),quality_level_ST=0.05, max_speed=20,nr_IQR_outlier=5,k=30, decl_grid=cfg_set["decl_grid_resid"],function=cfg_set["inter_fun_resid"], epsilon=cfg_set["epsilon_resid"],#factor_median=.2, return_single_vec=True,return_declust_vec=True, zero_interpol=cfg_set["zero_interpol"]) #UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,block_size_ST=15,winsize_LK5=(120,20),quality_level_ST=0.05, # max_speed=20,nr_IQR_outlier=5,decl_grid=20,function="inverse",k=20,factor_median=0.05, # return_single_vec=True,return_declust_vec=True,zero_interpol=True) #UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,block_size_ST=15,winsize_LK5=(120,20),quality_level_ST=0.05, # max_speed=20,nr_IQR_outlier=5,decl_grid=20,function="nearest",k=20,factor_median=.2, # return_single_vec=True,return_declust_vec=True) ## In case no motion vectors were detected, lower R_threshold by 20% if np.any(~np.isfinite(UV)): dBR_orig = dBR n_rep = 0 while np.any(~np.isfinite(UV)): if cfg_set["verbose"]: print(" *** Warning: No motion vectors detected, lower R_threshold by 30% to "+str(R_thresh*0.7)+" ***") R_thresh = R_thresh*0.7 dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_thresh) dBR[~np.isfinite(dBR)] = dBRmin if resid: oflow_source_data_min = oflow_source_data oflow_source_data[oflow_source_data<=R_thresh] = R_thresh oflow_source_data[~np.isfinite(oflow_source_data)] = R_thresh if not resid: UV, UV_vec, UV_vec_sp = oflow_method(dBR,return_single_vec=True,return_declust_vec=True) else: UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,winsize_LK5=(120,20),quality_level_ST=0.05, max_speed=20,nr_IQR_outlier=5,k=30, decl_grid=cfg_set["decl_grid_resid"],function=cfg_set["inter_fun_resid"], epsilon=cfg_set["epsilon_resid"],#factor_median=.2, return_single_vec=True,return_declust_vec=True, zero_interpol=cfg_set["zero_interpol"]) #UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,block_size_ST=15,winsize_LK5=(120,20),quality_level_ST=0.05, # max_speed=20,nr_IQR_outlier=5,decl_grid=20,function="inverse",k=20,epsilon=10,#factor_median=0.05, # return_single_vec=True,return_declust_vec=True,zero_interpol=True) #UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,block_size_ST=15,winsize_LK5=(120,20),quality_level_ST=0.05, # max_speed=20,nr_IQR_outlier=5,decl_grid=20,function="nearest",k=20,factor_median=.2, # return_single_vec=True,return_declust_vec=True) n_rep += 1 if n_rep > 2: UV = np.zeros((2,dBR.shape[1],dBR.shape[2])) if cfg_set["verbose"]: print(" *** Warning: Return zero UV-array! ***") break ## Invert direction of intermediate motion vectors #if cfg_set["UV_inter"]: # UV_vec[2:3,:,:] = -UV_vec[2:3,:,:] # UV_vec_sp[2:3,:,:] = -UV_vec_sp[2:3,:,:] """ ## Advect disp_test to get the advected test_array and the displacement array adv_method = st.advection.get_method(cfg_set["adv_method"]) dBR_adv, D = adv_method(dBR[-1,:,:], UV, 1, return_displacement=True) ## convert the forecasted dBR to mmhr if cfg_set["oflow_source"]=="RZC": if cfg_set["R_thresh_meth"] == "fix": R_tresh = cfg_set["R_threshold"] elif cfg_set["R_thresh_meth"] == "perc": R_tresh = np.min([np.nanpercentile(oflow_source_data[0,:,:],cfg_set["R_threshold"]), np.nanpercentile(oflow_source_data[1,:,:],cfg_set["R_threshold"])]) else: raise NotImplementedError("R_thresh_meth must either be set to 'fix' or 'perc'") oflow_source_data_forecast = st.utils.dBR2mmhr(dBR_adv, R_tresh) ## Print results: if False: calc_disparr_ctrl_plot(D,timestamps,oflow_source_data,oflow_source_data_forecast,cfg_set) """ #plt.imshow(D[0,:,:]) #plt.show() #fig, axes = plt.subplots(nrows=1, ncols=2) #fig1=axes[0].imshow(UV[0,:,:]) #fig.colorbar(fig1,ax=axes[0])#,orientation='horizontal') #fig2=axes[1].imshow(UV[1,:,:]) #fig.colorbar(fig2,ax=axes[1])#,orientation='horizontal') #fig.tight_layout() #plt.show() #sys.exit() if np.all(UV==0): #np.any(~np.isfinite(UV)): if cfg_set["instant_resid_corr"] and not resid: print(" *** Warning: No residual movement correction performed ***") D = UV.copy() else: adv_method = st.advection.get_method(cfg_set["adv_method"]) dBR_disp, D = adv_method(dBR[-2,:,:],UV,1,return_displacement=True,return_XYW=False) if cfg_set["instant_resid_corr"] and not resid: if cfg_set["verbose"]: print(" Make instantaneous residual movement correction") ## Advect second last observation to t0: dBR_disp[~np.isfinite(dBR_disp)] = dBRmin ## Convert dBR values of t0 and second last time step to mm/h: RZC_resid_fields = np.stack([st.utils.dBR2mmhr(dBR_disp[0,:,:], R_thresh), st.utils.dBR2mmhr(dBR[-1,:,:], R_thresh)]) #plt.imshow(RZC_resid_fields[0,:,:]); plt.title("RZC_resid_fields[0,:,:]"); plt.show() #plt.imshow(RZC_resid_fields[1,:,:]); plt.title("RZC_resid_fields[1,:,:]"); plt.show() ## Get residual displacement field UV_resid = oflow_method(RZC_resid_fields,min_distance_ST=2, winsize_LK5=(120,20),quality_level_ST=0.05, max_speed=20,nr_IQR_outlier=5,k=30, decl_grid=cfg_set["decl_grid_resid"],function=cfg_set["inter_fun_resid"], epsilon=cfg_set["epsilon_resid"],#factor_median=.2, zero_interpol=cfg_set["zero_interpol"]) ## Add UV_resid to original UV array n_rep = 0 while np.any(~np.isfinite(UV_resid)): print(" No UV_resid field found") R_thresh *= 0.7 RZC_resid_fields = np.stack([st.utils.dBR2mmhr(dBR_disp[0,:,:], R_thresh), st.utils.dBR2mmhr(dBR[-1,:,:], R_thresh)]) UV_resid = oflow_method(RZC_resid_fields,min_distance_ST=2, winsize_LK5=(120,20),quality_level_ST=0.05, max_speed=20,nr_IQR_outlier=5,k=30, decl_grid=cfg_set["decl_grid_resid"],function=cfg_set["inter_fun_resid"], epsilon=cfg_set["epsilon_resid"],#factor_median=.2, zero_interpol=cfg_set["zero_interpol"]) n_rep += 1 if n_rep > 2: UV_resid = np.zeros((2,dBR.shape[1],dBR.shape[2])) #if cfg_set["verbose"]: print(" *** Warning: Return zero UV_resid array! ***") break UV += UV_resid ## Displace with UV_resid field to get D_resid and add to D array: dBR_disp_disp, D = adv_method(RZC_resid_fields[0,:,:],UV,1, return_displacement=True,return_XYW=False) #D += D_resid if not cfg_set["UV_inter"]: return D, UV else: return D, UV, UV_vec, UV_vec_sp
def import_mch_metranet(filename, **kwargs): """Import a 8-bit bin radar reflectivity composite from the MeteoSwiss archive. Parameters ---------- filename : str Name of the file to import. Other Parameters ---------------- product : string The name of the MeteoSwiss QPE product: +------+----------------------------+ | Name | Product | +======+============================+ | AQC | Acquire | +------+----------------------------+ | CPC | CombiPrecip | +------+----------------------------+ | RZC | Precip | +------+----------------------------+ unit : string the physical unit of the data: 'mm/h', 'mm' or 'dBZ' accutime : float the accumulation time in minutes of the data Returns ------- out : tuple A three-element tuple containing the precipitation field in mm/h imported from a MeteoSwiss gif file and the associated quality field and metadata. The quality field is currently set to None. """ if not metranet_imported: raise Exception("metranet not imported") product = kwargs.get("product", "AQC") unit = kwargs.get("unit", "mm") accutime = kwargs.get("accutime", 5.) ret = metranet.read_file(filename, physic_value=True, verbose=False) R = ret.data geodata = _import_mch_geodata() # read metranet metadata = geodata metadata["institution"] = "MeteoSwiss" metadata["accutime"] = accutime metadata["unit"] = unit metadata["transform"] = None metadata["zerovalue"] = np.nanmin(R) if np.isnan(metadata["zerovalue"]): metadata["threshold"] = np.nan else: metadata["threshold"] = np.nanmin(R[R>metadata["zerovalue"]]) return R,None,metadata
def plot_oflow_derivation(cfg_set, t0, dt): ## Import datasets: #RZC = np.load("/data/COALITION2/PicturesSatellite/results_JMZ/2_input_NOSTRADAMUS_ANN/tmp/201507071830_RZC_orig.npy") #UVdisparr = np.load("/data/COALITION2/PicturesSatellite/results_JMZ/2_input_NOSTRADAMUS_ANN/tmp/201507071830_RZC_disparr_UV.npz") #Vx = UVdisparr["Vx"][0:,:,:]; Vy = UVdisparr["Vy"][0:,:,:] #UV_t0 = np.moveaxis(np.dstack((Vx[0,:,:],Vy[0,:,:])),2,0) #Dx = UVdisparr["Dx"]; Dy = UVdisparr["Dy"] ## Get index of respective RZC fields at t0 ind = (cfg_set["t_end_alt"] - t0).seconds / 60 / cfg_set["timestep"] ind2 = ind + (dt / cfg_set["timestep"]) ## Read in current oflow_source file: t_current = t0 cfg_set["timestep"] = dt filenames, timestamps = pth.path_creator(t_current, cfg_set["oflow_source"], "RADAR", cfg_set) ret = metranet.read_file(filenames[0], physic_value=True) oflow_source_data = np.atleast_3d(ret.data) for filename in filenames[1:]: ret_d_t = metranet.read_file(filename, physic_value=True) oflow_source_data_d_t = np.atleast_3d(ret_d_t.data) oflow_source_data = np.append(oflow_source_data, oflow_source_data_d_t, axis=2) oflow_source_data = np.moveaxis(oflow_source_data, 2, 0) #oflow_source_data_masked = np.ma.masked_invalid(oflow_source_data) #oflow_source_data_masked = np.ma.masked_where(oflow_source_data_masked==0,oflow_source_data_masked) ## convert linear rainrates to logarithimc dBR units if cfg_set["oflow_source"] == "RZC": if cfg_set["R_thresh_meth"] == "fix": R_tresh = cfg_set["R_threshold"] elif cfg_set["R_thresh_meth"] == "perc": R_tresh = np.min([ np.nanpercentile(oflow_source_data[0, :, :], cfg_set["R_threshold"]), np.nanpercentile(oflow_source_data[1, :, :], cfg_set["R_threshold"]) ]) else: raise ValueError( "R_thresh_meth must either be set to 'fix' or 'perc'") dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_tresh) dBR[~np.isfinite(dBR)] = dBRmin #R_thresh = cfg_set["R_threshold"] ## In case threshold is not exceeded, lower R_threshold by 20% while (dBR == dBRmin).all(): print( " *** Warning: Threshold not exceeded, lower R_threshold by 20% to " + str(R_thresh * 0.8) + " ***") R_thresh = R_thresh * 0.8 dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_thresh) dBR[~np.isfinite(dBR)] = dBRmin else: raise ValueError( "So far displacement array retrieval only implemented for RZC") ## Calculate UV field oflow_method = st.optflow.get_method(cfg_set["oflow_method_name"]) UV = oflow_method(dBR, return_single_vec=True)[1] UV_decl = oflow_method(dBR, return_declust_vec=True)[0] UV_final = oflow_method(dBR) #plt.imshow(oflow_source_data[0,:,:]) UV_final = np.stack([-UV_final[0, :, :], UV_final[1, :, :]]) print(UV.shape) print(UV_decl.shape) #fig, axs = plt.subplots(1,2, figsize=(10,6.5)) fig = plt.figure(figsize=(14, 9.5)) from matplotlib import gridspec gs = gridspec.GridSpec(1, 2, width_ratios=[1, 1, 1], wspace=0.0, hspace=0.0, top=0.95, bottom=0.05, left=0.05, right=0.95) if t0 == datetime.datetime(2015, 07, 07, 15, 00): xlimit = (270, 340) ylimit = (360, 380)
def get_vararr_t(t_current, var, cfg_set): """Get CCS4 variable array at timestep t_current. Parameters ---------- t_current : datetime object Current time for which to calculate displacement array. var : string Name of variable to be returned cfg_set : dict Basic variables defined in input_NOSTRADAMUS_ANN.py """ source = cfg_set["source_dict"][var] ## Implement different reader for different variable: if source == "RADAR": filenames, timestamps = pth.path_creator(t_current, var, source, cfg_set) index_timestep = np.where( [timestamp == t_current for timestamp in timestamps])[0][0] vararr = metranet.read_file(filenames[index_timestep], physic_value=True) #print(t_current,np.nanmax(vararr.data)) vararr = np.moveaxis(np.atleast_3d(vararr.data), 2, 0) return vararr elif source == "THX": filenames, timestamps = pth.path_creator(t_current, var, source, cfg_set) vararr = read_lightning_data(var, filenames, cfg_set, t_current) vararr = np.moveaxis(np.atleast_3d(vararr), 2, 0) #np.moveaxis(,2,1) return vararr elif source == "COSMO_WIND": filename, timestamps = pth.path_creator(t_current, var, source, cfg_set) vararr = read_wind_nc(filename) plt.imshow(vararr[0, :, :, :]) plt.show() sys.exit() return vararr elif source == "COSMO_CONV": if t_current.minute == 0: filename, timestamps = pth.path_creator(t_current, var, source, cfg_set) vararr = read_convection_nc(filename, var, cfg_set) else: filename_h_old, timestamp_h_old = pth.path_creator( t_current, var, source, cfg_set) vararr_old = read_convection_nc(filename_h_old, var, cfg_set) weight_old = 1 - t_current.minute / 60. t_current_plus1h = t_current + datetime.timedelta(hours=1) filename_h_new, timestamp_h_new = pth.path_creator( t_current_plus1h, var, source, cfg_set) vararr_new = read_convection_nc(filename_h_new, var, cfg_set) weight_new = 1 - weight_old vararr = weight_old * vararr_old + weight_new * vararr_new ## Smooth fields if requested (DEPRICATED): ## COSMO fields are smoothed before reading the statistics return vararr elif source == "SEVIRI": filenames, timestamps = pth.path_creator(t_current, var, source, cfg_set) if all(filename is None for filename in filenames): vararr = np.zeros( (1, cfg_set["xy_ext"][0], cfg_set["xy_ext"][1])) * np.nan else: vararr = read_sat_nc(filenames[0], cfg_set, var) vararr = np.moveaxis(np.atleast_3d(vararr), 2, 0) return vararr else: raise NotImplementedError("So far path_creator implemented \ RADAR, SEVIRI, COSMO_Conv, and THX variables only")