def read_convert_METRANET(filepath, ZDC=None): """This function reads in MeteoSwiss-Metranet-Files and converts values according to data-scale""" met_file = metranet.read_file(filepath) if met_file is None: return None data = met_file.data scale = met_file.scale if ZDC: scale = scale*200 converted = np.apply_along_axis(convert2scale, 1, data, scale) if ZDC: converted[np.where(converted < 600.)] = np.nan converted[np.where(converted > 6000.)] = np.nan return converted
def import_mch_metranet(filename, product, unit, accutime):
"""Import a 8-bit bin radar reflectivity composite from the MeteoSwiss
archive.
Parameters
----------
filename : str
Name of the file to import.
product : {"AQC", "CPC", "RZC", "AZC"}
The name of the MeteoSwiss QPE product.\n
Currently supported prducts:
+------+----------------------------+
| Name | Product |
+======+============================+
| AQC | Acquire |
+------+----------------------------+
| CPC | CombiPrecip |
+------+----------------------------+
| RZC | Precip |
+------+----------------------------+
| AZC | RZC accumulation |
+------+----------------------------+
unit : {"mm/h", "mm", "dBZ"}
the physical unit of the data
accutime : float
the accumulation time in minutes of the data
Returns
-------
out : tuple
A three-element tuple containing the precipitation field in mm/h imported
from a MeteoSwiss gif file and the associated quality field and metadata.
The quality field is currently set to None.
"""
if not metranet_imported:
raise MissingOptionalDependency(
"metranet package needed for importing MeteoSwiss "
"radar composites but it is not installed")
ret = metranet.read_file(filename, physic_value=True, verbose=False)
R = ret.data
geodata = _import_mch_geodata()
# read metranet
metadata = geodata
metadata["institution"] = "MeteoSwiss"
metadata["accutime"] = accutime
metadata["unit"] = unit
metadata["transform"] = None
metadata["zerovalue"] = np.nanmin(R)
if np.isnan(metadata["zerovalue"]):
metadata["threshold"] = np.nan
else:
metadata["threshold"] = np.nanmin(R[R > metadata["zerovalue"]])
metadata["zr_a"] = 316.0
metadata["zr_b"] = 1.5
return R, None, metadata
import sys
sys.path.append('/proj/lom/python/library/radar/io')
import metranet
radar_file="AZC1830312007L.812"
ret = metranet.read_file(radar_file, physic_value=True, verbose=True)
ret.data.tofile("AZC1830312007L.812.dat")
def calc_disparr(t_current, cfg_set, resid=False):
"""Get 2-dim displacement array for flow between timesteps t_current and t_current - n_past_frames*timestep.
Parameters
----------
t_current : datetime object
Current time for which to calculate displacement array.
cfg_set : dict
Basic variables defined in input_NOSTRADAMUS_ANN.py
resid : bool
Do displacement array creation for residual movement correction?
Default: False.
UV_inter : bool
If UV_inter is true, the calculated UV and sparsened UV vectors are returned as well.
See function check_create_disparray(t0, timestep, n_integ, root_path, t0_str, oflow_source, oflow_source_path)
"""
if resid:
## Read in current displaced oflow_source file:
filename = pth.path_creator_vararr("disp",cfg_set["oflow_source"],cfg_set)
#filename = "%stmp/%s_%s_disp%s.%s" % (cfg_set["root_path"], cfg_set["t0"].strftime("%Y%m%d%H%M"),
# cfg_set["oflow_source"], cfg_set["file_ext_verif"], cfg_set["save_type"])
t_diff = cfg_set["t0"] - t_current
t_diff_ind = int((t_diff.seconds/60)/cfg_set["timestep"])
#oflow_source_data = np.load(filename)[t_diff_ind:t_diff_ind+cfg_set["n_past_frames"]+1,:,:]
#oflow_source_data = np.load(filename)[t_diff_ind+cfg_set["n_past_frames_resid"]::-1,:,:][:cfg_set["n_past_frames"]+1]
oflow_source_data = iotmp.load_file(filename,cfg_set["oflow_source"])[t_diff_ind+cfg_set["n_past_frames_resid"]::-1,:,:][:cfg_set["n_past_frames"]+1]
if oflow_source_data.shape[0]==1:
UV = R = np.zeros((2,oflow_source_data.shape[1],oflow_source_data.shape[2]))
if not cfg_set["UV_inter"]: return UV, R
else: return UV, R, np.zeros(4)*np.nan, np.zeros(4)*np.nan
if np.all(np.array_equal(oflow_source_data[0,:,:],oflow_source_data[1,:,:])):
raise ValueError("Input data equal")
else:
## Read in current oflow_source file:
filenames, timestamps = pth.path_creator(t_current, cfg_set["oflow_source"], cfg_set["source_dict"][cfg_set["oflow_source"]], cfg_set)
ret = metranet.read_file(filenames[0], physic_value=True)
oflow_source_data = np.atleast_3d(ret.data)
for filename in filenames[1:]:
ret_d_t = metranet.read_file(filename, physic_value=True)
oflow_source_data_d_t = np.atleast_3d(ret_d_t.data)
oflow_source_data = np.append(oflow_source_data,oflow_source_data_d_t, axis=2)
oflow_source_data = np.moveaxis(oflow_source_data,2,0)
#oflow_source_data_masked = np.ma.masked_invalid(oflow_source_data)
#oflow_source_data_masked = np.ma.masked_where(oflow_source_data_masked==0,oflow_source_data_masked)
## Check whether there are non-nan entries:
if np.any(np.isnan(oflow_source_data).all(axis=(1,2))):
print(" *** Warning: Input oflow source field is all NAN!\n Returning NAN fields.***")
nan_arr = oflow_source_data[0,:,:]*np.nan
D = np.array([nan_arr,nan_arr])
UV = np.array([nan_arr,nan_arr])
UV_vec = []; UV_vec_sp = []
if not cfg_set["UV_inter"]:
return D, UV
else: return D, UV, UV_vec, UV_vec_sp
## Convert linear rainrates to logarithimc dBR units
if not cfg_set["oflow_source"]=="RZC":
raise NotImplementedError("So far displacement array retrieval only implemented for RZC")
else:
## Get threshold method:
if not resid:
R_thresh_meth = cfg_set["R_thresh_meth"]
R_threshold = cfg_set["R_threshold"]
else:
R_thresh_meth = cfg_set["R_thresh_meth_resid"]
R_threshold = cfg_set["R_threshold_resid"]
## Get threshold value:
if R_thresh_meth == "fix":
R_thresh = R_threshold
elif R_thresh_meth == "perc":
R_thresh = np.min([np.nanpercentile(oflow_source_data[0,:,:],R_threshold),
np.nanpercentile(oflow_source_data[1,:,:],R_threshold)])
else: raise ValueError("R_thresh_meth must either be set to 'fix' or 'perc'")
## Convert to dBR
dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_thresh)
dBR[~np.isfinite(dBR)] = dBRmin
#R_thresh = cfg_set["R_threshold"]
## In case threshold is not exceeded, lower R_threshold by 20%
while (dBR==dBRmin).all():
if cfg_set["verbose"]: print(" *** Warning: Threshold not exceeded, "+
"lower R_threshold by 20% to "+str(R_thresh*0.8)+" ***")
R_thresh = R_thresh*0.8
dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_thresh)
dBR[~np.isfinite(dBR)] = dBRmin
## For correction of residuals original mm/h values are used:
if resid:
oflow_source_data_min = oflow_source_data
oflow_source_data_min[oflow_source_data_min<=R_thresh] = R_thresh
oflow_source_data_min[~np.isfinite(oflow_source_data_min)] = R_thresh
## Calculate UV field
oflow_method = st.optflow.get_method(cfg_set["oflow_method_name"])
if not resid:
UV, UV_vec, UV_vec_sp = oflow_method(dBR,return_single_vec=True,return_declust_vec=True)
else:
UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,winsize_LK5=(120,20),quality_level_ST=0.05,
max_speed=20,nr_IQR_outlier=5,k=30,
decl_grid=cfg_set["decl_grid_resid"],function=cfg_set["inter_fun_resid"],
epsilon=cfg_set["epsilon_resid"],#factor_median=.2,
return_single_vec=True,return_declust_vec=True,
zero_interpol=cfg_set["zero_interpol"])
#UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,block_size_ST=15,winsize_LK5=(120,20),quality_level_ST=0.05,
# max_speed=20,nr_IQR_outlier=5,decl_grid=20,function="inverse",k=20,factor_median=0.05,
# return_single_vec=True,return_declust_vec=True,zero_interpol=True)
#UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,block_size_ST=15,winsize_LK5=(120,20),quality_level_ST=0.05,
# max_speed=20,nr_IQR_outlier=5,decl_grid=20,function="nearest",k=20,factor_median=.2,
# return_single_vec=True,return_declust_vec=True)
## In case no motion vectors were detected, lower R_threshold by 20%
if np.any(~np.isfinite(UV)):
dBR_orig = dBR
n_rep = 0
while np.any(~np.isfinite(UV)):
if cfg_set["verbose"]:
print(" *** Warning: No motion vectors detected, lower R_threshold by 30% to "+str(R_thresh*0.7)+" ***")
R_thresh = R_thresh*0.7
dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_thresh)
dBR[~np.isfinite(dBR)] = dBRmin
if resid:
oflow_source_data_min = oflow_source_data
oflow_source_data[oflow_source_data<=R_thresh] = R_thresh
oflow_source_data[~np.isfinite(oflow_source_data)] = R_thresh
if not resid:
UV, UV_vec, UV_vec_sp = oflow_method(dBR,return_single_vec=True,return_declust_vec=True)
else:
UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,winsize_LK5=(120,20),quality_level_ST=0.05,
max_speed=20,nr_IQR_outlier=5,k=30,
decl_grid=cfg_set["decl_grid_resid"],function=cfg_set["inter_fun_resid"],
epsilon=cfg_set["epsilon_resid"],#factor_median=.2,
return_single_vec=True,return_declust_vec=True,
zero_interpol=cfg_set["zero_interpol"])
#UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,block_size_ST=15,winsize_LK5=(120,20),quality_level_ST=0.05,
# max_speed=20,nr_IQR_outlier=5,decl_grid=20,function="inverse",k=20,epsilon=10,#factor_median=0.05,
# return_single_vec=True,return_declust_vec=True,zero_interpol=True)
#UV, UV_vec, UV_vec_sp = oflow_method(oflow_source_data_min,min_distance_ST=2,block_size_ST=15,winsize_LK5=(120,20),quality_level_ST=0.05,
# max_speed=20,nr_IQR_outlier=5,decl_grid=20,function="nearest",k=20,factor_median=.2,
# return_single_vec=True,return_declust_vec=True)
n_rep += 1
if n_rep > 2:
UV = np.zeros((2,dBR.shape[1],dBR.shape[2]))
if cfg_set["verbose"]: print(" *** Warning: Return zero UV-array! ***")
break
## Invert direction of intermediate motion vectors
#if cfg_set["UV_inter"]:
# UV_vec[2:3,:,:] = -UV_vec[2:3,:,:]
# UV_vec_sp[2:3,:,:] = -UV_vec_sp[2:3,:,:]
"""
## Advect disp_test to get the advected test_array and the displacement array
adv_method = st.advection.get_method(cfg_set["adv_method"])
dBR_adv, D = adv_method(dBR[-1,:,:], UV, 1, return_displacement=True)
## convert the forecasted dBR to mmhr
if cfg_set["oflow_source"]=="RZC":
if cfg_set["R_thresh_meth"] == "fix":
R_tresh = cfg_set["R_threshold"]
elif cfg_set["R_thresh_meth"] == "perc":
R_tresh = np.min([np.nanpercentile(oflow_source_data[0,:,:],cfg_set["R_threshold"]),
np.nanpercentile(oflow_source_data[1,:,:],cfg_set["R_threshold"])])
else: raise NotImplementedError("R_thresh_meth must either be set to 'fix' or 'perc'")
oflow_source_data_forecast = st.utils.dBR2mmhr(dBR_adv, R_tresh)
## Print results:
if False:
calc_disparr_ctrl_plot(D,timestamps,oflow_source_data,oflow_source_data_forecast,cfg_set)
"""
#plt.imshow(D[0,:,:])
#plt.show()
#fig, axes = plt.subplots(nrows=1, ncols=2)
#fig1=axes[0].imshow(UV[0,:,:])
#fig.colorbar(fig1,ax=axes[0])#,orientation='horizontal')
#fig2=axes[1].imshow(UV[1,:,:])
#fig.colorbar(fig2,ax=axes[1])#,orientation='horizontal')
#fig.tight_layout()
#plt.show()
#sys.exit()
if np.all(UV==0): #np.any(~np.isfinite(UV)):
if cfg_set["instant_resid_corr"] and not resid:
print(" *** Warning: No residual movement correction performed ***")
D = UV.copy()
else:
adv_method = st.advection.get_method(cfg_set["adv_method"])
dBR_disp, D = adv_method(dBR[-2,:,:],UV,1,return_displacement=True,return_XYW=False)
if cfg_set["instant_resid_corr"] and not resid:
if cfg_set["verbose"]:
print(" Make instantaneous residual movement correction")
## Advect second last observation to t0:
dBR_disp[~np.isfinite(dBR_disp)] = dBRmin
## Convert dBR values of t0 and second last time step to mm/h:
RZC_resid_fields = np.stack([st.utils.dBR2mmhr(dBR_disp[0,:,:], R_thresh),
st.utils.dBR2mmhr(dBR[-1,:,:], R_thresh)])
#plt.imshow(RZC_resid_fields[0,:,:]); plt.title("RZC_resid_fields[0,:,:]"); plt.show()
#plt.imshow(RZC_resid_fields[1,:,:]); plt.title("RZC_resid_fields[1,:,:]"); plt.show()
## Get residual displacement field
UV_resid = oflow_method(RZC_resid_fields,min_distance_ST=2,
winsize_LK5=(120,20),quality_level_ST=0.05,
max_speed=20,nr_IQR_outlier=5,k=30,
decl_grid=cfg_set["decl_grid_resid"],function=cfg_set["inter_fun_resid"],
epsilon=cfg_set["epsilon_resid"],#factor_median=.2,
zero_interpol=cfg_set["zero_interpol"])
## Add UV_resid to original UV array
n_rep = 0
while np.any(~np.isfinite(UV_resid)):
print(" No UV_resid field found")
R_thresh *= 0.7
RZC_resid_fields = np.stack([st.utils.dBR2mmhr(dBR_disp[0,:,:], R_thresh),
st.utils.dBR2mmhr(dBR[-1,:,:], R_thresh)])
UV_resid = oflow_method(RZC_resid_fields,min_distance_ST=2,
winsize_LK5=(120,20),quality_level_ST=0.05,
max_speed=20,nr_IQR_outlier=5,k=30,
decl_grid=cfg_set["decl_grid_resid"],function=cfg_set["inter_fun_resid"],
epsilon=cfg_set["epsilon_resid"],#factor_median=.2,
zero_interpol=cfg_set["zero_interpol"])
n_rep += 1
if n_rep > 2:
UV_resid = np.zeros((2,dBR.shape[1],dBR.shape[2]))
#if cfg_set["verbose"]:
print(" *** Warning: Return zero UV_resid array! ***")
break
UV += UV_resid
## Displace with UV_resid field to get D_resid and add to D array:
dBR_disp_disp, D = adv_method(RZC_resid_fields[0,:,:],UV,1,
return_displacement=True,return_XYW=False)
#D += D_resid
if not cfg_set["UV_inter"]:
return D, UV
else: return D, UV, UV_vec, UV_vec_sp
def import_mch_metranet(filename, **kwargs):
"""Import a 8-bit bin radar reflectivity composite from the MeteoSwiss
archive.
Parameters
----------
filename : str
Name of the file to import.
Other Parameters
----------------
product : string
The name of the MeteoSwiss QPE product:
+------+----------------------------+
| Name | Product |
+======+============================+
| AQC | Acquire |
+------+----------------------------+
| CPC | CombiPrecip |
+------+----------------------------+
| RZC | Precip |
+------+----------------------------+
unit : string
the physical unit of the data: 'mm/h', 'mm' or 'dBZ'
accutime : float
the accumulation time in minutes of the data
Returns
-------
out : tuple
A three-element tuple containing the precipitation field in mm/h imported
from a MeteoSwiss gif file and the associated quality field and metadata.
The quality field is currently set to None.
"""
if not metranet_imported:
raise Exception("metranet not imported")
product = kwargs.get("product", "AQC")
unit = kwargs.get("unit", "mm")
accutime = kwargs.get("accutime", 5.)
ret = metranet.read_file(filename, physic_value=True, verbose=False)
R = ret.data
geodata = _import_mch_geodata()
# read metranet
metadata = geodata
metadata["institution"] = "MeteoSwiss"
metadata["accutime"] = accutime
metadata["unit"] = unit
metadata["transform"] = None
metadata["zerovalue"] = np.nanmin(R)
if np.isnan(metadata["zerovalue"]):
metadata["threshold"] = np.nan
else:
metadata["threshold"] = np.nanmin(R[R>metadata["zerovalue"]])
return R,None,metadata
def plot_oflow_derivation(cfg_set, t0, dt):
## Import datasets:
#RZC = np.load("/data/COALITION2/PicturesSatellite/results_JMZ/2_input_NOSTRADAMUS_ANN/tmp/201507071830_RZC_orig.npy")
#UVdisparr = np.load("/data/COALITION2/PicturesSatellite/results_JMZ/2_input_NOSTRADAMUS_ANN/tmp/201507071830_RZC_disparr_UV.npz")
#Vx = UVdisparr["Vx"][0:,:,:]; Vy = UVdisparr["Vy"][0:,:,:]
#UV_t0 = np.moveaxis(np.dstack((Vx[0,:,:],Vy[0,:,:])),2,0)
#Dx = UVdisparr["Dx"]; Dy = UVdisparr["Dy"]
## Get index of respective RZC fields at t0
ind = (cfg_set["t_end_alt"] - t0).seconds / 60 / cfg_set["timestep"]
ind2 = ind + (dt / cfg_set["timestep"])
## Read in current oflow_source file:
t_current = t0
cfg_set["timestep"] = dt
filenames, timestamps = pth.path_creator(t_current,
cfg_set["oflow_source"], "RADAR",
cfg_set)
ret = metranet.read_file(filenames[0], physic_value=True)
oflow_source_data = np.atleast_3d(ret.data)
for filename in filenames[1:]:
ret_d_t = metranet.read_file(filename, physic_value=True)
oflow_source_data_d_t = np.atleast_3d(ret_d_t.data)
oflow_source_data = np.append(oflow_source_data,
oflow_source_data_d_t,
axis=2)
oflow_source_data = np.moveaxis(oflow_source_data, 2, 0)
#oflow_source_data_masked = np.ma.masked_invalid(oflow_source_data)
#oflow_source_data_masked = np.ma.masked_where(oflow_source_data_masked==0,oflow_source_data_masked)
## convert linear rainrates to logarithimc dBR units
if cfg_set["oflow_source"] == "RZC":
if cfg_set["R_thresh_meth"] == "fix":
R_tresh = cfg_set["R_threshold"]
elif cfg_set["R_thresh_meth"] == "perc":
R_tresh = np.min([
np.nanpercentile(oflow_source_data[0, :, :],
cfg_set["R_threshold"]),
np.nanpercentile(oflow_source_data[1, :, :],
cfg_set["R_threshold"])
])
else:
raise ValueError(
"R_thresh_meth must either be set to 'fix' or 'perc'")
dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_tresh)
dBR[~np.isfinite(dBR)] = dBRmin
#R_thresh = cfg_set["R_threshold"]
## In case threshold is not exceeded, lower R_threshold by 20%
while (dBR == dBRmin).all():
print(
" *** Warning: Threshold not exceeded, lower R_threshold by 20% to "
+ str(R_thresh * 0.8) + " ***")
R_thresh = R_thresh * 0.8
dBR, dBRmin = st.utils.mmhr2dBR(oflow_source_data, R_thresh)
dBR[~np.isfinite(dBR)] = dBRmin
else:
raise ValueError(
"So far displacement array retrieval only implemented for RZC")
## Calculate UV field
oflow_method = st.optflow.get_method(cfg_set["oflow_method_name"])
UV = oflow_method(dBR, return_single_vec=True)[1]
UV_decl = oflow_method(dBR, return_declust_vec=True)[0]
UV_final = oflow_method(dBR)
#plt.imshow(oflow_source_data[0,:,:])
UV_final = np.stack([-UV_final[0, :, :], UV_final[1, :, :]])
print(UV.shape)
print(UV_decl.shape)
#fig, axs = plt.subplots(1,2, figsize=(10,6.5))
fig = plt.figure(figsize=(14, 9.5))
from matplotlib import gridspec
gs = gridspec.GridSpec(1,
2,
width_ratios=[1, 1, 1],
wspace=0.0,
hspace=0.0,
top=0.95,
bottom=0.05,
left=0.05,
right=0.95)
if t0 == datetime.datetime(2015, 07, 07, 15, 00):
xlimit = (270, 340)
ylimit = (360, 380)
def get_vararr_t(t_current, var, cfg_set):
"""Get CCS4 variable array at timestep t_current.
Parameters
----------
t_current : datetime object
Current time for which to calculate displacement array.
var : string
Name of variable to be returned
cfg_set : dict
Basic variables defined in input_NOSTRADAMUS_ANN.py
"""
source = cfg_set["source_dict"][var]
## Implement different reader for different variable:
if source == "RADAR":
filenames, timestamps = pth.path_creator(t_current, var, source,
cfg_set)
index_timestep = np.where(
[timestamp == t_current for timestamp in timestamps])[0][0]
vararr = metranet.read_file(filenames[index_timestep],
physic_value=True)
#print(t_current,np.nanmax(vararr.data))
vararr = np.moveaxis(np.atleast_3d(vararr.data), 2, 0)
return vararr
elif source == "THX":
filenames, timestamps = pth.path_creator(t_current, var, source,
cfg_set)
vararr = read_lightning_data(var, filenames, cfg_set, t_current)
vararr = np.moveaxis(np.atleast_3d(vararr), 2, 0) #np.moveaxis(,2,1)
return vararr
elif source == "COSMO_WIND":
filename, timestamps = pth.path_creator(t_current, var, source,
cfg_set)
vararr = read_wind_nc(filename)
plt.imshow(vararr[0, :, :, :])
plt.show()
sys.exit()
return vararr
elif source == "COSMO_CONV":
if t_current.minute == 0:
filename, timestamps = pth.path_creator(t_current, var, source,
cfg_set)
vararr = read_convection_nc(filename, var, cfg_set)
else:
filename_h_old, timestamp_h_old = pth.path_creator(
t_current, var, source, cfg_set)
vararr_old = read_convection_nc(filename_h_old, var, cfg_set)
weight_old = 1 - t_current.minute / 60.
t_current_plus1h = t_current + datetime.timedelta(hours=1)
filename_h_new, timestamp_h_new = pth.path_creator(
t_current_plus1h, var, source, cfg_set)
vararr_new = read_convection_nc(filename_h_new, var, cfg_set)
weight_new = 1 - weight_old
vararr = weight_old * vararr_old + weight_new * vararr_new
## Smooth fields if requested (DEPRICATED):
## COSMO fields are smoothed before reading the statistics
return vararr
elif source == "SEVIRI":
filenames, timestamps = pth.path_creator(t_current, var, source,
cfg_set)
if all(filename is None for filename in filenames):
vararr = np.zeros(
(1, cfg_set["xy_ext"][0], cfg_set["xy_ext"][1])) * np.nan
else:
vararr = read_sat_nc(filenames[0], cfg_set, var)
vararr = np.moveaxis(np.atleast_3d(vararr), 2, 0)
return vararr
else:
raise NotImplementedError("So far path_creator implemented \
RADAR, SEVIRI, COSMO_Conv, and THX variables only")