def main():
"""
"""
# parse the arguments
parser = argparse.ArgumentParser(
description='Entry to Pyrad processing framework')
# positional arguments
parser.add_argument('proc_cfgfile',
type=str,
help='name of main configuration file')
parser.add_argument('days',
nargs='+',
type=str,
help='Dates to process. Format YYYY-MM-DD')
# keyword arguments
parser.add_argument('--trtbase',
type=str,
default='/store/msrad/radar/trt/',
help='name of folder containing the TRT cell data')
parser.add_argument(
'--radarbase',
type=str,
default='/store/msrad/radar/pyrad_products/rad4alp_hydro_PHA/',
help='name of folder containing the radar data')
parser.add_argument('--cfgpath',
type=str,
default=os.path.expanduser('~') +
'/pyrad/config/processing/',
help='configuration file path')
parser.add_argument(
'--datatypes',
type=str,
default='hydro,KDPc,dBZc,RhoHVc,TEMP,ZDRc',
help='Name of the polarimetric moments to process. Coma separated')
parser.add_argument(
'--datasets',
type=str,
default='hydroclass,KDPc,reflectivity,RhoHVc,temperature,ZDRc',
help='Name of the directory containing the datasets')
parser.add_argument('--hres',
type=float,
default=250.,
help='Height resolution')
args = parser.parse_args()
print("====== PYRAD TRT data processing started: %s" %
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
atexit.register(_print_end_msg,
"====== PYRAD TRT data processing finished: ")
print('config path: ' + args.cfgpath)
print('config file: ' + args.proc_cfgfile)
print('trt path: ' + args.trtbase)
print('radar data path: ' + args.radarbase)
cfgfile_proc = args.cfgpath + args.proc_cfgfile
trajtype = 'trt'
time_dir_list = args.days
datatype_list = args.datatypes.split(',')
dataset_list = args.datasets.split(',')
if np.size(datatype_list) != np.size(dataset_list):
warn(
str(np.size(datatype_list)) + ' datatypes but ' +
str(np.size(dataset_list)) +
' dataset directories. Their number must be equal')
return
# Find all TRT files in directory
trt_list = []
for time_dir in time_dir_list:
trt_list.extend(
glob.glob(args.trtbase + time_dir + '/TRTC_cell_plots/All/*.trt'))
trt_list.extend(
glob.glob(args.trtbase + time_dir + '/TRTC_cell_plots/Some/*.trt'))
# Pyrad data processing
trt_cell_id_list = []
trt_file_list = []
for fname in trt_list:
print('processing TRT cell file ' + fname)
try:
infostr = os.path.basename(fname).split('.')[0]
pyrad_main(cfgfile_proc,
trajfile=fname,
infostr=infostr,
trajtype=trajtype)
trt_cell_id_list.append(infostr)
trt_file_list.append(fname)
except ValueError:
print(ValueError)
# plot time series and get altitude of graupel column
if 'hydro' in datatype_list:
cell_ID_list = np.asarray([], dtype=int)
time_list = np.asarray([], dtype=datetime.datetime)
lon_list = np.asarray([], dtype=float)
lat_list = np.asarray([], dtype=float)
area_list = np.asarray([], dtype=float)
rank_list = np.asarray([], dtype=float)
rm_hmin_list = np.ma.asarray([], dtype=float)
rm_hmax_list = np.ma.asarray([], dtype=float)
for i, trt_cell_id in enumerate(trt_cell_id_list):
print('\n\nPost-processing cell: ' + trt_cell_id)
dt_str = trt_cell_id[0:12]
dt_cell = datetime.datetime.strptime(dt_str, "%Y%m%d%H%M")
time_dir = dt_cell.strftime("%Y-%m-%d")
for j, datatype in enumerate(datatype_list):
dataset = dataset_list[j]
file_base2 = args.radarbase + time_dir + '/' + dataset + '_trt_traj/'
field_name = get_fieldname_pyart(datatype)
field_dict = get_metadata(field_name)
titl = 'TRT cell ' + trt_cell_id + '\n' + get_field_name(
field_dict, field_name)
# plot time-height
flist = glob.glob(file_base2 + 'PROFILE/*_' + trt_cell_id +
'_rhi_profile_*_' + datatype + '_hres' +
str(int(args.hres)) + '.csv')
if not flist:
warn('No profile files found in ' + file_base2 +
'PROFILE/ for TRT cell ' + trt_cell_id +
' with resolution ' + str(args.hres))
else:
labels = [
'50.0-percentile', '25.0-percentile', '75.0-percentile'
]
if datatype == 'RhoHVc':
labels = [
'80.0-percentile', '65.0-percentile', '95.0-percentile'
]
elif datatype == 'hydro':
labels = [
'Mode', '2nd most common', '3rd most common',
'% points mode', '% points 2nd most common',
'% points 3rd most common'
]
elif datatype == 'entropy' or 'prop' in datatype:
labels = ['Mean', 'Min', 'Max']
tbin_edges, hbin_edges, _, data_ma, start_time = (
read_profile_ts(flist, labels, hres=args.hres))
basepath_out = os.path.dirname(flist[0])
fname = (basepath_out + '/' + trt_cell_id +
'_trt_TIME_HEIGHT_' + datatype + '_hres' +
str(args.hres) + '.png')
vmin = vmax = None
if datatype == 'RhoHVc':
vmin = 0.95
vmax = 1.00
xlabel = ('time (s from ' +
start_time.strftime("%Y-%m-%d %H:%M:%S") + ')')
_plot_time_range(tbin_edges,
hbin_edges,
data_ma,
field_name, [fname],
titl=titl,
xlabel=xlabel,
ylabel='height (m MSL)',
figsize=[10, 8],
vmin=vmin,
vmax=vmax,
dpi=72)
print("----- plot to '%s'" % fname)
# Get min and max altitude of graupel/hail area
if datatype == 'hydro':
(traj_ID, yyyymmddHHMM, lon, lat, _, _, _, area, _, _, _,
RANKr, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_) = read_trt_traj_data(trt_file_list[i])
hmin, hmax = get_graupel_column(tbin_edges, hbin_edges,
data_ma, start_time,
yyyymmddHHMM)
cell_ID_list = np.append(cell_ID_list, traj_ID)
time_list = np.append(time_list, yyyymmddHHMM)
lon_list = np.append(lon_list, lon)
lat_list = np.append(lat_list, lat)
area_list = np.append(area_list, area)
rank_list = np.append(rank_list, RANKr)
rm_hmin_list = np.ma.append(rm_hmin_list, hmin)
rm_hmax_list = np.ma.append(rm_hmax_list, hmax)
# plot time-hist
flist = glob.glob(file_base2 + 'HISTOGRAM/*_' + trt_cell_id +
'_histogram_*_' + datatype + '.csv')
if not flist:
warn('No histogram files found in ' + file_base2 +
'HISTOGRAM/ for TRT cell ' + trt_cell_id)
else:
tbin_edges, bin_edges, data_ma, start_time = read_histogram_ts(
flist, datatype)
basepath_out = os.path.dirname(flist[0])
fname = (basepath_out + '/' + trt_cell_id + '_trt_HISTOGRAM_' +
datatype + '.png')
data_ma[data_ma == 0.] = np.ma.masked
xlabel = ('time (s from ' +
start_time.strftime("%Y-%m-%d %H:%M:%S") + ')')
_plot_time_range(tbin_edges,
bin_edges,
data_ma,
'frequency_of_occurrence', [fname],
titl=titl,
xlabel=xlabel,
ylabel=get_colobar_label(
field_dict, field_name),
vmin=0.,
vmax=np.max(data_ma),
figsize=[10, 8],
dpi=72)
print("----- plot to '%s'" % fname)
# plot quantiles
flist = glob.glob(file_base2 + 'QUANTILES/*_' + trt_cell_id +
'_quantiles_*_' + datatype + '.csv')
if not flist:
warn('No quantiles files found in ' + file_base2 +
'QUANTILES/ for TRT cell ' + trt_cell_id)
continue
tbin_edges, qbin_edges, data_ma, start_time = read_quantiles_ts(
flist, step=5., qmin=0., qmax=100.)
basepath_out = os.path.dirname(flist[0])
fname = (basepath_out + '/' + trt_cell_id + '_trt_QUANTILES_' +
datatype + '.png')
vmin = vmax = None
if datatype == 'RhoHVc':
vmin = 0.95
vmax = 1.00
xlabel = ('time (s from ' +
start_time.strftime("%Y-%m-%d %H:%M:%S") + ')')
_plot_time_range(tbin_edges,
qbin_edges,
data_ma,
field_name, [fname],
titl=titl,
xlabel=xlabel,
ylabel='Quantile',
vmin=vmin,
vmax=vmax,
figsize=[10, 8],
dpi=72)
print("----- plot to '%s'" % fname)
if 'hydro' in datatype_list:
fname = args.trtbase + 'cell_rimmed_particles_column.csv'
write_trt_cell_lightning(cell_ID_list, time_list, lon_list, lat_list,
area_list, rank_list, rm_hmin_list,
rm_hmax_list, fname)
print("----- written to '%s'" % fname)
def main():
"""
"""
# parse the arguments
parser = argparse.ArgumentParser(
description='Entry to Pyrad processing framework')
# positional arguments
parser.add_argument(
'proc_cfgfile', type=str, help='name of main configuration file')
parser.add_argument(
'starttime', type=str,
help=('starting time of the data to be processed. ' +
'Format ''YYYYMMDDhhmmss'''))
parser.add_argument(
'endtime', type=str,
help='end time of the data to be processed. Format ''YYYYMMDDhhmmss''')
# keyword arguments
parser.add_argument(
'--cfgpath', type=str,
default=os.path.expanduser('~')+'/pyrad/config/processing/',
help='configuration file path')
parser.add_argument(
'--storepath', type=str,
default='/store/msrad/radar/pyrad_products/rad4alp_birds_PHA/',
help='Base data storing path')
parser.add_argument(
'--hres', type=int, default=200, help='Height resolution [m]')
args = parser.parse_args()
print("====== PYRAD data processing started: %s" %
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
atexit.register(_print_end_msg,
"====== PYRAD data processing finished: ")
print('config path: '+args.cfgpath)
print('config file: '+args.proc_cfgfile)
print('start time: '+args.starttime)
print('end time: '+args.endtime)
proc_starttime = datetime.datetime.strptime(
args.starttime, '%Y%m%d%H%M%S')
proc_endtime = datetime.datetime.strptime(
args.endtime, '%Y%m%d%H%M%S')
cfgfile_proc = args.cfgpath+args.proc_cfgfile
pyrad_main(cfgfile_proc, starttime=proc_starttime, endtime=proc_endtime)
# Plot time-height
file_base = args.storepath
hres = args.hres
datatype_list = [
'dBZc', 'eta_h', 'bird_density', 'WIND_SPEED', 'WIND_DIRECTION',
'wind_vel_h_u', 'wind_vel_h_v', 'wind_vel_v']
startdate = proc_starttime.replace(hour=0, minute=0, second=0, microsecond=0)
enddate = proc_endtime.replace(hour=0, minute=0, second=0, microsecond=0)
ndays = int((enddate-startdate).days)+1
for datatype in datatype_list:
flist = []
for i in range(ndays):
time_dir = (
proc_starttime+datetime.timedelta(days=i)).strftime('%Y-%m-%d')
filepath = (
file_base+time_dir+'/VAD/PROFILE_WIND/' +
'*_wind_profile_VAD_WIND_hres'+str(hres)+'.csv')
labels = [
'u_wind', 'std_u_wind', 'np_u_wind',
'v_wind', 'std_v_wind', 'np_v_wind',
'w_wind', 'std_w_wind', 'np_w_wind',
'mag_h_wind', 'dir_h_wind']
label_nr = 0
if datatype == 'dBZc':
filepath = (
file_base+time_dir+'/velFilter/PROFILE_dBZc/' +
'*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
labels = [
'50.0-percentile', '25.0-percentile', '75.0-percentile']
# dBZ mean data
# filepath = (
# file_base+time_dir+'/velFilter/PROFILE_dBZc_mean/' +
# '*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
# dBZ linear mean data
# filepath = (
# file_base+time_dir+'/velFilter/PROFILE_dBZc_linear_mean/' +
# '*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
# dBZ before filtering with fitted velocity
# filepath = (
# file_base+time_dir+'/echoFilter/PROFILE_dBZc/' +
# '*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
# labels = [
# '50.0-percentile', '25.0-percentile', '75.0-percentile']
#
# dBZ before filtering with fitted velocity. Linear mean
# filepath = (
# file_base+time_dir+'/echoFilter/PROFILE_dBZc_linear_mean/' +
# '*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
elif datatype == 'eta_h':
filepath = (
file_base+time_dir+'/vol_refl/PROFILE/' +
'*_rhi_profile_*_eta_h_hres'+str(hres)+'.csv')
labels = [
'50.0-percentile', '25.0-percentile', '75.0-percentile']
# mean data
# filepath = (
# file_base+time_dir+'/vol_refl/PROFILE_mean/' +
# '*_rhi_profile_*_eta_h_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
elif datatype == 'bird_density':
filepath = (
file_base+time_dir+'/bird_density/PROFILE/' +
'*_rhi_profile_*_bird_density_hres'+str(hres)+'.csv')
labels = [
'50.0-percentile', '25.0-percentile', '75.0-percentile']
# mean data
# filepath = (
# file_base+time_dir+'/bird_density/PROFILE_mean/' +
# '*_rhi_profile_*_bird_density_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
elif datatype == 'WIND_SPEED':
label_nr = 9
elif datatype == 'WIND_DIRECTION':
label_nr = 10
elif datatype == 'wind_vel_h_u':
label_nr = 0
elif datatype == 'wind_vel_h_v':
label_nr = 3
elif datatype == 'wind_vel_v':
label_nr = 6
flist_aux = glob.glob(filepath)
if not flist_aux:
warn('No profile files found in '+filepath)
continue
flist.extend(flist_aux)
if not flist:
warn('No profile files found')
continue
flist.sort()
field_name = get_fieldname_pyart(datatype)
field_dict = get_metadata(field_name)
titl = 'bird retrieval '+args.starttime+'\n'+get_field_name(
field_dict, field_name)
tbin_edges, hbin_edges, np_ma, data_ma, t_start = read_profile_ts(
flist, labels, hres=hres, label_nr=label_nr)
basepath_out = os.path.dirname(flist[0])
fname = (
basepath_out+'/'+args.starttime+'_TIME_HEIGHT_' +
datatype+'_hres'+str(hres)+'.png')
vmin = vmax = None
_plot_time_range(
tbin_edges, hbin_edges/1000., data_ma, field_name, [fname],
titl=titl, figsize=[10, 8], vmin=vmin, vmax=vmax, dpi=72)
print("----- plot to '%s'" % fname)
# Plot number of points
field_dict = get_metadata('number_of_samples')
titl = 'bird retrieval '+args.starttime+'\n'+get_field_name(
field_dict, 'number_of_samples')
fname = (
basepath_out+'/'+args.starttime+'_TIME_HEIGHT_' +
datatype+'nsamples_hres'+str(hres)+'.png')
vmin = vmax = None
_plot_time_range(
tbin_edges, hbin_edges/1000., np_ma, 'number_of_samples', [fname],
titl=titl, figsize=[10, 8], vmin=vmin, vmax=vmax, dpi=72)
print("----- plot to '%s'" % fname)
def main():
"""
"""
file_base = '/store/msrad/radar/pyrad_products/rad4alp_hydro_PHA/'
time_dir_list = ['2017-06-29']
trt_cell_id = '2017062913000174'
hres = 250
datatype_list = ['dBZc', 'ZDRc', 'RhoHVc', 'KDPc', 'TEMP', 'hydro']
dataset_list = [
'reflectivity', 'ZDRc', 'RhoHVc', 'KDPc', 'temperature', 'hydroclass'
]
print("====== Plot time-height started: %s" %
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
atexit.register(_print_end_msg, "====== Plot time-height finished: ")
for time_dir in time_dir_list:
for i, datatype in enumerate(datatype_list):
labels = ['50.0-percentile', '25.0-percentile', '75.0-percentile']
if datatype == 'RhoHVc':
labels = [
'80.0-percentile', '65.0-percentile', '95.0-percentile'
]
elif datatype == 'hydro':
labels = [
'Mode', '2nd most common', '3rd most common',
'% points mode', '% points 2nd most common',
'% points 3rd most common'
]
dataset = dataset_list[i]
file_path = file_base + time_dir + '/' + dataset + '_trt_traj/PROFILE/'
flist = glob.glob(file_path + '*_' + trt_cell_id +
'_rhi_profile_*_' + datatype + '_hres' +
str(hres) + '.csv')
if not flist:
warn('No profile files found in ' + file_path +
' for TRT cell ' + trt_cell_id + ' with resolution ' +
str(hres))
continue
tbin_edges, hbin_edges, data_ma = read_profile_ts(flist,
labels,
hres=hres)
basepath_out = os.path.dirname(flist[0])
fname = (basepath_out + '/' + trt_cell_id + '_trt_TIME_HEIGHT_' +
datatype + '_hres' + str(hres) + '.png')
field_name = get_fieldname_pyart(datatype)
field_dict = get_metadata(field_name)
titl = 'TRT cell ' + trt_cell_id + '\n' + get_field_name(
field_dict, field_name)
vmin = vmax = None
if datatype == 'RhoHVc':
vmin = 0.95
vmax = 1.00
_plot_time_range(tbin_edges,
hbin_edges,
data_ma,
field_name, [fname],
titl=titl,
figsize=[10, 8],
vmin=vmin,
vmax=vmax,
dpi=72)
print("----- plot to '%s'" % fname)