def nc_workflow(dir_in,
dir_out,
fname_supl='IKA_vprhi',
overwrite=False,
**kws):
"""Extract profiles and save as nc."""
n_hbins = 297
files = np.sort(glob(path.join(dir_in, "*RHI_HV*.raw")))
time_filename = path.basename(files[0])[0:8]
fileOut = path.join(dir_out, time_filename + '_' + fname_supl + '.nc')
vps = dict()
if path.exists(fileOut) and not overwrite:
print('{} [notice] file already exists, skipping.'.format(fileOut))
return
for file_indx, filename in enumerate(files):
print(filename)
try: # reading radar data
radar = pyart.io.read(filename)
except Exception as e:
eprint('{fname} [read error] {e}'.format(fname=filename, e=e))
continue
ts, df = rhi2vp(radar, n_hbins=n_hbins, **kws)
vps[ts] = df
df = pd.concat(vps)
df.index.rename(['time', 'height'], inplace=True)
ds = df.to_xarray()
if dir_out is not None:
ds.to_netcdf(fileOut)
return ds
def mat_workflow(dir_in,
dir_out,
fname_supl='IKA_vprhi',
overwrite=False,
**kws):
"""LEGACY method to extract profiles and save as mat."""
n_hbins = 297
files = np.sort(glob(path.join(dir_in, "*RHI_HV*.raw")))
ObsTime = []
time_filename = path.basename(files[0])[0:8]
fileOut = path.join(dir_out, time_filename + '_' + fname_supl + '.mat')
if path.exists(fileOut) and not overwrite:
print('{} [notice] file already exists, skipping.'.format(fileOut))
return
for file_indx, filename in enumerate(files):
print(filename)
try: # reading radar data
radar = pyart.io.read(filename)
except Exception as e:
eprint('{fname} [read error] {e}'.format(fname=filename, e=e))
continue
# TODO: very much broken after this line
ts, df = rhi2vp(radar, n_hbins=n_hbins, **kws)
if ts is None:
raise ValueError('no timestamp')
tstr = path.basename(filename)[0:12]
ObsTime.append(datetime.strptime(tstr, '%Y%m%d%H%M').isoformat())
print(fileOut)
vp_rhi = {'ObsTime': ObsTime, 'height': df.index.values}
vp_rhi.update(df.to_dict(orient='list'))
sio.savemat(fileOut, {'VP_RHI': vp_rhi})
return df # for debugging
def kdp_maesaka(radar, **kws):
"""Compute KDP using Maesaka algo from a radar object."""
mask = radar.fields['differential_phase']['data'].mask
try:
kdp_m = pyart.retrieve.kdp_maesaka(radar, **kws)
except IndexError:
# outlier checking sometimes causes trouble (with weak kdp?)
eprint('Skipping outlier check.')
kdp_m = pyart.retrieve.kdp_maesaka(radar, check_outliers=False, **kws)
return np.ma.masked_array(data=kdp_m[0]['data'], mask=mask)
def rhi_preprocess(radar, r_poi=R_IKA_HYDE, r_agg=1e3, **kws):
"""Process RHI data for aggregation."""
try: # extracting variables
fix_elevation(radar)
rdr_vars = extract_radar_vars(radar, **kws)
except Exception as e:
eprint('[extract error] {e}'.format(e=e))
return None, None
r = radar.gate_x['data'] # horizontal range
rvars = filter_range(rdr_vars, r, r_poi, r_agg)
hght = height(radar, r, r_poi)
return rvars, hght
def kdp_recalc_if_missing(fpath, verbose=True, force_recalc=False,
invert_mask=False):
"""Recalculate KDP from PHIDP."""
radar = pyart.io.read_cfradial(fpath)
try:
kdp, phif, phir = pyart.retrieve.kdp_maesaka(radar, psidp_field='PHIDP')
except IndexError:
eprint('IndexError. Skipping {}.'.format(fpath))
return
if verbose:
print(fpath)
kdp_old = radar.get_field(0, 'KDP')
# if none of kdp values make sense
if not (abs(kdp_old < 1)).any() or force_recalc:
mask_phi = radar.get_field(0, 'PHIDP').mask
if invert_mask:
# some data had an inverted mask, invert it back
mask_kdp = ~kdp_old.mask + mask_phi # inverted kdp mask
else:
mask_kdp = kdp_old.mask + mask_phi
try:
kdp['data'] = np.ma.masked_array(kdp['data'], mask=mask_kdp,
fill_value=kdp_old.fill_value)
except np.ma.core.MaskError:
eprint('MaskError. Skipping {}.'.format(fpath))
return
radar.add_field('KDP', kdp, replace_existing=True)
pyart.io.write_cfradial(fpath, radar) # overwrite changes
if verbose:
eprint('KDP recalculated. File rewritten.')
elif verbose:
eprint('KDP already present. File left untouched.')
def batch_process(name,
data=None,
write_tif=True,
write_png=True,
resultsdir=None,
interpolate=True,
verbose=False):
rc = data['rain_c']
tstr = data['tim']
tparser = lambda ts: datetime.datetime.strptime(ts[0][0], '%Y%m%d_%H%M%S')
t = map(tparser, tstr)
tr = map(round_datetime, t)
n_timesteps = rc.shape[2]
rs = [rc[:, :, i] for i in range(n_timesteps)]
i = 0
for j, (r0, r1, t0,
t1) in enumerate(itertools.izip(rs, rs[1:], tr, tr[1:])):
if interpolate:
dt = t1 - t0
dt_minutes = int(dt.total_seconds() / 60)
if dt_minutes > 5:
eprint('Long interpolation period of {} minutes.'.format(
dt_minutes))
ri = interpolation.interp(r0, r1, n=dt_minutes - 1)
ra = [r0] + ri
else:
ra = [r0]
for extra_minutes, r in enumerate(ra):
i += 1
tim = t0 + datetime.timedelta(minutes=extra_minutes)
if verbose:
print(str(tim))
basename = tim.strftime('%Y%m%d_%H%M')
if write_png:
fig, axdd = plot_frame(r, j, tim, data=data)
framedir = ensure_join(resultsdir, name, 'png')
framepath = path.join(framedir, basename + '.png')
fig.savefig(framepath)
plt.close(fig)
if write_tif:
tifdir = ensure_join(resultsdir, name, 'tif')
tif_out_fpath = path.join(tifdir, basename + '.tif')
meta = {'TIFFTAG_DATETIME': str(tim)}
savetif(r.astype(rasterio.float32), meta, tif_out_fpath)
def sigmet2cfrad(filepath,
outdir='cf',
dirlock=None,
verbose=False,
filter_raw=False):
"""Convert Sigmet raw radar data to cfradial format"""
radar = io.read_sigmet(filepath)
if is_bad(radar):
if filter_raw:
if verbose:
eprint('rm {}'.format(filepath))
remove(filepath)
return
time = radar_start_datetime(radar)
if dirlock:
dirlock.acquire()
subdir = ensure_dir(path.join(outdir, time.strftime('%Y%m%d')))
if dirlock:
dirlock.release()
out_filepath = path.join(subdir, cfrad_filename(radar))
if verbose:
print(out_filepath)
io.write_cfradial(out_filepath, radar)
def filter_filepaths(filepaths_all, remove_bad=False, verbose=False):
"""skip or remove filepaths that trigger bad_postprocessing_result"""
filepaths_good = copy.deepcopy(filepaths_all)
for filepath in filepaths_all:
try:
with nc.Dataset(filepath, 'r') as ncdata:
if bad_postprocessing_result(list(ncdata.variables)):
filepaths_good.remove(filepath)
if remove_bad:
remove(filepath)
if verbose:
eprint('rm ' + filepath)
elif verbose:
eprint('filtered ' + filepath)
except FileNotFoundError:
eprint('Filtering unfound file {}.'.format(filepath))
filepaths_good.remove(filepath)
return filepaths_good