def main():
"""
"""
# parse the arguments
parser = argparse.ArgumentParser(
description='Entry to Pyrad processing framework')
# positional arguments
parser.add_argument('proc_cfgfile',
type=str,
help='name of main configuration file')
# keyword arguments
parser.add_argument('--starttime',
type=str,
default=None,
help=('starting time of the data to be processed. ' +
'Format '
'YYYYMMDDhhmmss'
''))
parser.add_argument('--endtime',
type=str,
default=None,
help='end time of the data to be processed. Format '
'YYYYMMDDhhmmss'
'')
parser.add_argument('--postproc_cfgfile',
type=str,
default=None,
help='name of main post-processing configuration file')
parser.add_argument('--cfgpath',
type=str,
default=os.path.expanduser('~') +
'/pyrad/config/processing/',
help='configuration file path')
parser.add_argument("-i",
"--infostr",
type=str,
help="Information string about the actual data "
"processing (e.g. 'RUN57'). This string is added "
"to the filenames of the product files.",
default="")
parser.add_argument("-t",
"--trajfile",
type=str,
default='',
help="Definition file of plane trajectory. "
"Configuration of scan sector, products, ...")
parser.add_argument("--trajtype",
type=str,
default='plane',
help="Type of trajectory. "
"Can be either 'plane', 'lightning' or 'proc_periods'")
parser.add_argument("--flashnr",
type=int,
default=0,
help="If type of trajectory is 'lightning', "
"flash number the data of which will be processed"
"0 means that all lightning data will be processed")
parser.add_argument("--MULTIPROCESSING_DSET",
type=int,
default=0,
help="If 1 the generation of the datasets at the "
"same processing level will be parallelized")
parser.add_argument("--MULTIPROCESSING_PROD",
type=int,
default=0,
help="If 1 the generation of the products of each "
"dataset will be parallelized")
parser.add_argument("--PROFILE_MULTIPROCESSING",
type=int,
default=0,
help="If 1 the multiprocessing is profiled")
args = parser.parse_args()
print("====== PYRAD data processing started: %s" %
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
atexit.register(_print_end_msg, "====== PYRAD data processing finished: ")
print('config path: ' + args.cfgpath)
print('config file: ' + args.proc_cfgfile)
print('postproc config file: ' + str(args.postproc_cfgfile))
if args.starttime is not None:
print('start time: ' + args.starttime)
else:
print('start time not defined by user')
if args.endtime is not None:
print('end time: ' + args.endtime)
else:
print('end time not defined by user')
if args.MULTIPROCESSING_DSET:
print('Dataset generation will be parallelized')
if args.MULTIPROCESSING_PROD:
print('Product generation will be parallelized')
if args.PROFILE_MULTIPROCESSING:
print('Parallel processing performance will be profiled')
proc_starttime = None
if args.starttime is not None:
proc_starttime = datetime.datetime.strptime(args.starttime,
'%Y%m%d%H%M%S')
proc_endtime = None
if args.endtime is not None:
proc_endtime = datetime.datetime.strptime(args.endtime, '%Y%m%d%H%M%S')
cfgfile_proc = args.cfgpath + args.proc_cfgfile
if args.infostr == 'None':
infostr = ''
else:
infostr = args.infostr
pyrad_main(cfgfile_proc,
starttime=proc_starttime,
endtime=proc_endtime,
trajfile=args.trajfile,
infostr=infostr,
trajtype=args.trajtype,
flashnr=args.flashnr,
MULTIPROCESSING_DSET=args.MULTIPROCESSING_DSET,
MULTIPROCESSING_PROD=args.MULTIPROCESSING_PROD,
PROFILE_MULTIPROCESSING=args.PROFILE_MULTIPROCESSING)
if args.postproc_cfgfile is not None:
cfgfile_postproc = args.cfgpath + args.postproc_cfgfile
pyrad_main(cfgfile_postproc,
starttime=proc_starttime,
endtime=proc_endtime,
trajfile=args.trajfile,
infostr=infostr,
trajtype=args.trajtype,
flashnr=args.flashnr,
MULTIPROCESSING_DSET=args.MULTIPROCESSING_DSET,
MULTIPROCESSING_PROD=args.MULTIPROCESSING_PROD,
PROFILE_MULTIPROCESSING=args.PROFILE_MULTIPROCESSING)
def main():
"""
"""
# parse the arguments
parser = argparse.ArgumentParser(
description='Entry to Pyrad processing framework')
# positional arguments
parser.add_argument('proc_cfgfile',
type=str,
help='name of main configuration file')
parser.add_argument('days',
nargs='+',
type=str,
help='Dates to process. Format YYYY-MM-DD')
# keyword arguments
parser.add_argument('--trtbase',
type=str,
default='/store/msrad/radar/trt/',
help='name of folder containing the TRT cell data')
parser.add_argument(
'--radarbase',
type=str,
default='/store/msrad/radar/pyrad_products/rad4alp_hydro_PHA/',
help='name of folder containing the radar data')
parser.add_argument('--cfgpath',
type=str,
default=os.path.expanduser('~') +
'/pyrad/config/processing/',
help='configuration file path')
parser.add_argument(
'--datatypes',
type=str,
default='hydro,KDPc,dBZc,RhoHVc,TEMP,ZDRc',
help='Name of the polarimetric moments to process. Coma separated')
parser.add_argument(
'--datasets',
type=str,
default='hydroclass,KDPc,reflectivity,RhoHVc,temperature,ZDRc',
help='Name of the directory containing the datasets')
parser.add_argument('--hres',
type=float,
default=250.,
help='Height resolution')
args = parser.parse_args()
print("====== PYRAD TRT data processing started: %s" %
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
atexit.register(_print_end_msg,
"====== PYRAD TRT data processing finished: ")
print('config path: ' + args.cfgpath)
print('config file: ' + args.proc_cfgfile)
print('trt path: ' + args.trtbase)
print('radar data path: ' + args.radarbase)
cfgfile_proc = args.cfgpath + args.proc_cfgfile
trajtype = 'trt'
time_dir_list = args.days
datatype_list = args.datatypes.split(',')
dataset_list = args.datasets.split(',')
if np.size(datatype_list) != np.size(dataset_list):
warn(
str(np.size(datatype_list)) + ' datatypes but ' +
str(np.size(dataset_list)) +
' dataset directories. Their number must be equal')
return
# Find all TRT files in directory
trt_list = []
for time_dir in time_dir_list:
trt_list.extend(
glob.glob(args.trtbase + time_dir + '/TRTC_cell_plots/All/*.trt'))
trt_list.extend(
glob.glob(args.trtbase + time_dir + '/TRTC_cell_plots/Some/*.trt'))
# Pyrad data processing
trt_cell_id_list = []
trt_file_list = []
for fname in trt_list:
print('processing TRT cell file ' + fname)
try:
infostr = os.path.basename(fname).split('.')[0]
pyrad_main(cfgfile_proc,
trajfile=fname,
infostr=infostr,
trajtype=trajtype)
trt_cell_id_list.append(infostr)
trt_file_list.append(fname)
except ValueError:
print(ValueError)
# plot time series and get altitude of graupel column
if 'hydro' in datatype_list:
cell_ID_list = np.asarray([], dtype=int)
time_list = np.asarray([], dtype=datetime.datetime)
lon_list = np.asarray([], dtype=float)
lat_list = np.asarray([], dtype=float)
area_list = np.asarray([], dtype=float)
rank_list = np.asarray([], dtype=float)
rm_hmin_list = np.ma.asarray([], dtype=float)
rm_hmax_list = np.ma.asarray([], dtype=float)
for i, trt_cell_id in enumerate(trt_cell_id_list):
print('\n\nPost-processing cell: ' + trt_cell_id)
dt_str = trt_cell_id[0:12]
dt_cell = datetime.datetime.strptime(dt_str, "%Y%m%d%H%M")
time_dir = dt_cell.strftime("%Y-%m-%d")
for j, datatype in enumerate(datatype_list):
dataset = dataset_list[j]
file_base2 = args.radarbase + time_dir + '/' + dataset + '_trt_traj/'
field_name = get_fieldname_pyart(datatype)
field_dict = get_metadata(field_name)
titl = 'TRT cell ' + trt_cell_id + '\n' + get_field_name(
field_dict, field_name)
# plot time-height
flist = glob.glob(file_base2 + 'PROFILE/*_' + trt_cell_id +
'_rhi_profile_*_' + datatype + '_hres' +
str(int(args.hres)) + '.csv')
if not flist:
warn('No profile files found in ' + file_base2 +
'PROFILE/ for TRT cell ' + trt_cell_id +
' with resolution ' + str(args.hres))
else:
labels = [
'50.0-percentile', '25.0-percentile', '75.0-percentile'
]
if datatype == 'RhoHVc':
labels = [
'80.0-percentile', '65.0-percentile', '95.0-percentile'
]
elif datatype == 'hydro':
labels = [
'Mode', '2nd most common', '3rd most common',
'% points mode', '% points 2nd most common',
'% points 3rd most common'
]
elif datatype == 'entropy' or 'prop' in datatype:
labels = ['Mean', 'Min', 'Max']
tbin_edges, hbin_edges, _, data_ma, start_time = (
read_profile_ts(flist, labels, hres=args.hres))
basepath_out = os.path.dirname(flist[0])
fname = (basepath_out + '/' + trt_cell_id +
'_trt_TIME_HEIGHT_' + datatype + '_hres' +
str(args.hres) + '.png')
vmin = vmax = None
if datatype == 'RhoHVc':
vmin = 0.95
vmax = 1.00
xlabel = ('time (s from ' +
start_time.strftime("%Y-%m-%d %H:%M:%S") + ')')
_plot_time_range(tbin_edges,
hbin_edges,
data_ma,
field_name, [fname],
titl=titl,
xlabel=xlabel,
ylabel='height (m MSL)',
figsize=[10, 8],
vmin=vmin,
vmax=vmax,
dpi=72)
print("----- plot to '%s'" % fname)
# Get min and max altitude of graupel/hail area
if datatype == 'hydro':
(traj_ID, yyyymmddHHMM, lon, lat, _, _, _, area, _, _, _,
RANKr, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
_) = read_trt_traj_data(trt_file_list[i])
hmin, hmax = get_graupel_column(tbin_edges, hbin_edges,
data_ma, start_time,
yyyymmddHHMM)
cell_ID_list = np.append(cell_ID_list, traj_ID)
time_list = np.append(time_list, yyyymmddHHMM)
lon_list = np.append(lon_list, lon)
lat_list = np.append(lat_list, lat)
area_list = np.append(area_list, area)
rank_list = np.append(rank_list, RANKr)
rm_hmin_list = np.ma.append(rm_hmin_list, hmin)
rm_hmax_list = np.ma.append(rm_hmax_list, hmax)
# plot time-hist
flist = glob.glob(file_base2 + 'HISTOGRAM/*_' + trt_cell_id +
'_histogram_*_' + datatype + '.csv')
if not flist:
warn('No histogram files found in ' + file_base2 +
'HISTOGRAM/ for TRT cell ' + trt_cell_id)
else:
tbin_edges, bin_edges, data_ma, start_time = read_histogram_ts(
flist, datatype)
basepath_out = os.path.dirname(flist[0])
fname = (basepath_out + '/' + trt_cell_id + '_trt_HISTOGRAM_' +
datatype + '.png')
data_ma[data_ma == 0.] = np.ma.masked
xlabel = ('time (s from ' +
start_time.strftime("%Y-%m-%d %H:%M:%S") + ')')
_plot_time_range(tbin_edges,
bin_edges,
data_ma,
'frequency_of_occurrence', [fname],
titl=titl,
xlabel=xlabel,
ylabel=get_colobar_label(
field_dict, field_name),
vmin=0.,
vmax=np.max(data_ma),
figsize=[10, 8],
dpi=72)
print("----- plot to '%s'" % fname)
# plot quantiles
flist = glob.glob(file_base2 + 'QUANTILES/*_' + trt_cell_id +
'_quantiles_*_' + datatype + '.csv')
if not flist:
warn('No quantiles files found in ' + file_base2 +
'QUANTILES/ for TRT cell ' + trt_cell_id)
continue
tbin_edges, qbin_edges, data_ma, start_time = read_quantiles_ts(
flist, step=5., qmin=0., qmax=100.)
basepath_out = os.path.dirname(flist[0])
fname = (basepath_out + '/' + trt_cell_id + '_trt_QUANTILES_' +
datatype + '.png')
vmin = vmax = None
if datatype == 'RhoHVc':
vmin = 0.95
vmax = 1.00
xlabel = ('time (s from ' +
start_time.strftime("%Y-%m-%d %H:%M:%S") + ')')
_plot_time_range(tbin_edges,
qbin_edges,
data_ma,
field_name, [fname],
titl=titl,
xlabel=xlabel,
ylabel='Quantile',
vmin=vmin,
vmax=vmax,
figsize=[10, 8],
dpi=72)
print("----- plot to '%s'" % fname)
if 'hydro' in datatype_list:
fname = args.trtbase + 'cell_rimmed_particles_column.csv'
write_trt_cell_lightning(cell_ID_list, time_list, lon_list, lat_list,
area_list, rank_list, rm_hmin_list,
rm_hmax_list, fname)
print("----- written to '%s'" % fname)
def main():
"""
"""
# parse the arguments
parser = argparse.ArgumentParser(
description='Entry to Pyrad processing framework')
# positional arguments
parser.add_argument(
'proc_cfgfile', type=str, help='name of main configuration file')
parser.add_argument(
'starttime', type=str,
help=('starting time of the data to be processed. ' +
'Format ''YYYYMMDDhhmmss'''))
parser.add_argument(
'endtime', type=str,
help='end time of the data to be processed. Format ''YYYYMMDDhhmmss''')
# keyword arguments
parser.add_argument(
'--cfgpath', type=str,
default=os.path.expanduser('~')+'/pyrad/config/processing/',
help='configuration file path')
parser.add_argument(
'--storepath', type=str,
default='/store/msrad/radar/pyrad_products/rad4alp_birds_PHA/',
help='Base data storing path')
parser.add_argument(
'--hres', type=int, default=200, help='Height resolution [m]')
args = parser.parse_args()
print("====== PYRAD data processing started: %s" %
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
atexit.register(_print_end_msg,
"====== PYRAD data processing finished: ")
print('config path: '+args.cfgpath)
print('config file: '+args.proc_cfgfile)
print('start time: '+args.starttime)
print('end time: '+args.endtime)
proc_starttime = datetime.datetime.strptime(
args.starttime, '%Y%m%d%H%M%S')
proc_endtime = datetime.datetime.strptime(
args.endtime, '%Y%m%d%H%M%S')
cfgfile_proc = args.cfgpath+args.proc_cfgfile
pyrad_main(cfgfile_proc, starttime=proc_starttime, endtime=proc_endtime)
# Plot time-height
file_base = args.storepath
hres = args.hres
datatype_list = [
'dBZc', 'eta_h', 'bird_density', 'WIND_SPEED', 'WIND_DIRECTION',
'wind_vel_h_u', 'wind_vel_h_v', 'wind_vel_v']
startdate = proc_starttime.replace(hour=0, minute=0, second=0, microsecond=0)
enddate = proc_endtime.replace(hour=0, minute=0, second=0, microsecond=0)
ndays = int((enddate-startdate).days)+1
for datatype in datatype_list:
flist = []
for i in range(ndays):
time_dir = (
proc_starttime+datetime.timedelta(days=i)).strftime('%Y-%m-%d')
filepath = (
file_base+time_dir+'/VAD/PROFILE_WIND/' +
'*_wind_profile_VAD_WIND_hres'+str(hres)+'.csv')
labels = [
'u_wind', 'std_u_wind', 'np_u_wind',
'v_wind', 'std_v_wind', 'np_v_wind',
'w_wind', 'std_w_wind', 'np_w_wind',
'mag_h_wind', 'dir_h_wind']
label_nr = 0
if datatype == 'dBZc':
filepath = (
file_base+time_dir+'/velFilter/PROFILE_dBZc/' +
'*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
labels = [
'50.0-percentile', '25.0-percentile', '75.0-percentile']
# dBZ mean data
# filepath = (
# file_base+time_dir+'/velFilter/PROFILE_dBZc_mean/' +
# '*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
# dBZ linear mean data
# filepath = (
# file_base+time_dir+'/velFilter/PROFILE_dBZc_linear_mean/' +
# '*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
# dBZ before filtering with fitted velocity
# filepath = (
# file_base+time_dir+'/echoFilter/PROFILE_dBZc/' +
# '*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
# labels = [
# '50.0-percentile', '25.0-percentile', '75.0-percentile']
#
# dBZ before filtering with fitted velocity. Linear mean
# filepath = (
# file_base+time_dir+'/echoFilter/PROFILE_dBZc_linear_mean/' +
# '*_rhi_profile_*_dBZc_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
elif datatype == 'eta_h':
filepath = (
file_base+time_dir+'/vol_refl/PROFILE/' +
'*_rhi_profile_*_eta_h_hres'+str(hres)+'.csv')
labels = [
'50.0-percentile', '25.0-percentile', '75.0-percentile']
# mean data
# filepath = (
# file_base+time_dir+'/vol_refl/PROFILE_mean/' +
# '*_rhi_profile_*_eta_h_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
elif datatype == 'bird_density':
filepath = (
file_base+time_dir+'/bird_density/PROFILE/' +
'*_rhi_profile_*_bird_density_hres'+str(hres)+'.csv')
labels = [
'50.0-percentile', '25.0-percentile', '75.0-percentile']
# mean data
# filepath = (
# file_base+time_dir+'/bird_density/PROFILE_mean/' +
# '*_rhi_profile_*_bird_density_hres'+str(hres)+'.csv')
# labels = [
# 'Mean', 'Min', 'Max']
elif datatype == 'WIND_SPEED':
label_nr = 9
elif datatype == 'WIND_DIRECTION':
label_nr = 10
elif datatype == 'wind_vel_h_u':
label_nr = 0
elif datatype == 'wind_vel_h_v':
label_nr = 3
elif datatype == 'wind_vel_v':
label_nr = 6
flist_aux = glob.glob(filepath)
if not flist_aux:
warn('No profile files found in '+filepath)
continue
flist.extend(flist_aux)
if not flist:
warn('No profile files found')
continue
flist.sort()
field_name = get_fieldname_pyart(datatype)
field_dict = get_metadata(field_name)
titl = 'bird retrieval '+args.starttime+'\n'+get_field_name(
field_dict, field_name)
tbin_edges, hbin_edges, np_ma, data_ma, t_start = read_profile_ts(
flist, labels, hres=hres, label_nr=label_nr)
basepath_out = os.path.dirname(flist[0])
fname = (
basepath_out+'/'+args.starttime+'_TIME_HEIGHT_' +
datatype+'_hres'+str(hres)+'.png')
vmin = vmax = None
_plot_time_range(
tbin_edges, hbin_edges/1000., data_ma, field_name, [fname],
titl=titl, figsize=[10, 8], vmin=vmin, vmax=vmax, dpi=72)
print("----- plot to '%s'" % fname)
# Plot number of points
field_dict = get_metadata('number_of_samples')
titl = 'bird retrieval '+args.starttime+'\n'+get_field_name(
field_dict, 'number_of_samples')
fname = (
basepath_out+'/'+args.starttime+'_TIME_HEIGHT_' +
datatype+'nsamples_hres'+str(hres)+'.png')
vmin = vmax = None
_plot_time_range(
tbin_edges, hbin_edges/1000., np_ma, 'number_of_samples', [fname],
titl=titl, figsize=[10, 8], vmin=vmin, vmax=vmax, dpi=72)
print("----- plot to '%s'" % fname)
def main():
"""
"""
# parse the arguments
parser = argparse.ArgumentParser(
description='Entry to Pyrad processing framework')
# positional arguments
parser.add_argument('cfgfiles',
nargs='+',
type=str,
help='name of main configuration file')
# keyword arguments
parser.add_argument('--starttime',
type=str,
default=None,
help=('starting time of the data to be processed. ' +
'Format '
'YYYYMMDDhhmmss'
''))
parser.add_argument('--endtime',
type=str,
default=None,
help='end time of the data to be processed. Format '
'YYYYMMDDhhmmss'
'')
parser.add_argument('--cfgpath',
type=str,
default=os.path.expanduser('~') +
'/pyrad/config/processing/',
help='configuration file path')
parser.add_argument('--proc_period',
type=int,
default=60,
help='Period between processing rounds (s)')
parser.add_argument('--proc_finish',
type=int,
default=None,
help='Processing time allowed before shutdown (s)')
args = parser.parse_args()
print("====== PYRAD data processing started: %s" %
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
atexit.register(_print_end_msg, "====== PYRAD data processing finished: ")
print('config path: ' + args.cfgpath)
cfgfile_list = []
for ind, cfgfile in enumerate(args.cfgfiles):
print('config file ' + str(ind) + ': ' + cfgfile)
cfgfile_list.append(args.cfgpath + cfgfile)
if args.starttime is not None:
print('start time: ' + args.starttime)
else:
print('start time not defined by user')
if args.endtime is not None:
print('end time: ' + args.endtime)
else:
print('end time not defined by user')
proc_starttime = None
if args.starttime is not None:
proc_starttime = datetime.datetime.strptime(args.starttime,
'%Y%m%d%H%M%S')
proc_endtime = None
if args.endtime is not None:
proc_endtime = datetime.datetime.strptime(args.endtime, '%Y%m%d%H%M%S')
end_proc = False
while not end_proc:
try:
end_proc = pyrad_main(cfgfile_list,
starttime=proc_starttime,
endtime=proc_endtime,
proc_period=args.proc_period,
proc_finish=args.proc_finish)
except:
traceback.print_exc()
if args.proc_finish is None:
warn("An exception occurred. " +
"Restarting the real time processing")
else:
end_proc = True
