def export_spectra_to_nc(data, system='', path='', **kwargs):
"""
This routine generates an hourly NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'.
Args:
data (dict): dictionary of larda containers
system (string): name of the radar system
path (string): path where the NetCDF file is stored
"""
no_chirps = len(data['rg_offsets']) - 1
dt_start = h.ts_to_dt(data['ts'][0])
dt_end = h.ts_to_dt(data['ts'][-1])
ds_name = path + '{}-{}_{}_spectra.nc'.format(dt_start.strftime("%Y%m%d-%H%M"), dt_end.strftime("%H%M"), system)
print('open file: ', ds_name)
with netCDF4.Dataset(ds_name, "w", format="NETCDF4") as ds:
# ds.commit_id = subprocess.check_output(["git", "describe", "--always"]).rstrip()
ds.description = '{} calibrated Doppler spectra'.format(system)
ds.history = 'Created ' + time.ctime(time.time())
ds.system = system
ds.location = data['paraminfo']['location']
ds._FillValue = data['paraminfo']['fill_value']
ds.createDimension('chirp', no_chirps) # add variable number of chirps later
ds.createDimension('time', data['ts'].size)
for ic in range(no_chirps):
ds.createDimension(f'C{ic + 1}range', data['rg'][ic].size)
ds.createDimension(f'C{ic + 1}velocity', data['vel'][ic].size)
nc_add_variable(ds, val=data['paraminfo']['coordinates'][0], dimension=(),
var_name='latitude', type=np.float32, long_name='GPS latitude', unit='deg')
nc_add_variable(ds, val=data['paraminfo']['coordinates'][1], dimension=(),
var_name='longitude', type=np.float32, long_name='GPS longitude', unit='deg')
nc_add_variable(ds, val=data['ts'], dimension=('time',),
var_name='time', type=np.float64, long_name='Unix Time - seconds since 01.01.1970 00:00 UTC', unit='sec')
nc_add_variable(ds, val=data['rg_offsets'][:no_chirps], dimension=('chirp',),
var_name='rg_offsets', type=np.float32, long_name='Range Indices when Chirp shifts ', unit='-')
for ic in range(no_chirps):
nc_add_variable(ds, val=data['rg'][ic], dimension=(f'C{ic + 1}range',),
var_name=f'C{ic + 1}range', type=np.float32, long_name='range', unit='m')
nc_add_variable(ds, val=data['vel'][ic], dimension=(f'C{ic + 1}velocity',),
var_name=f'C{ic + 1}vel', type=np.float32, long_name='velocity', unit='m s-1')
nc_add_variable(ds, val=data['var'][ic], dimension=('time', f'C{ic + 1}range', f'C{ic + 1}velocity'),
var_name=f'C{ic + 1}Zspec', type=np.float32,
long_name=f'Doppler spectrum at vertical+horizontal polarization: Chirp {ic + 1}', unit='mm6 m-3')
nc_add_variable(ds, val=data['vel'][ic][-1], dimension=('chirp',),
var_name=f'C{ic + 1}DoppMax', type=np.float32, long_name='Unambiguous Doppler velocity (+/-)', unit='m s-1')
return 0
def get_time_slicer(ts, f, time_interval):
"""get time slicer from the time_interval
Following options are available
1. time_interval with [ts_begin, ts_end]
2. only one timestamp is selected and the found
right one would be beyond the ts range -> argnearest instead searchsorted
3. only one is timestamp
"""
# select first timestamp right of begin (not left if nearer as above)
#print(f'start time {h.ts_to_dt(ts[0])}')
it_b = 0 if ts.shape[0] == 1 else np.searchsorted(
ts, h.dt_to_ts(time_interval[0]), side='right')
if len(time_interval) == 2:
it_e = h.argnearest(ts, h.dt_to_ts(time_interval[1]))
if it_b == ts.shape[0]: it_b = it_b - 1
valid_step = 3 * np.median(np.diff(ts))
if ts[it_e] < h.dt_to_ts(
time_interval[0]) - valid_step or ts[it_b] < h.dt_to_ts(
time_interval[0]):
# second condition is to ensure that no timestamp before
# the selected interval is choosen
# (problem with limrad after change of sampling frequency)
str = 'found last profile of file {}\n at ts[it_e] {} too far ({}s) from {}\n'.format(
f, h.ts_to_dt(ts[it_e]), valid_step, time_interval[0]) \
+ 'or begin too early {} < {}\n returning None'.format(h.ts_to_dt(ts[it_b]), time_interval[0])
logger.warning(str)
return None
it_e = it_e + 1 if not it_e == ts.shape[0] - 1 else None
slicer = [slice(it_b, it_e)]
elif it_b == ts.shape[0]:
# only one timestamp is selected
# and the found right one would be beyond the ts range
it_b = h.argnearest(ts, h.dt_to_ts(time_interval[0]))
slicer = [slice(it_b, it_b + 1)]
else:
slicer = [slice(it_b, it_b + 1)]
return slicer
def to_text(data_cont):
"""represent the tree as a multiline string"""
maxind_at_level = np.cumsum([2**n for n in range(5)]) - 1
single_tree = data_cont['var']
no_levels = np.searchsorted(maxind_at_level, max(single_tree.keys()))
print("no levels", no_levels)
dt = h.ts_to_dt(data_cont['ts'])
lines = []
lines.append("tree at ts {} and rg {:.2f}".format(
dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3], data_cont['rg']))
header = "id (bounds) " + (no_levels)*" " + \
" Z v σ 𝛾"
if single_tree[0]['ldr'] != -99:
header += " ldr ldrmax"
header += " thres prom"
lines.append(header)
# iterate over children depth first, to build a proper tree
for i in child_iter(list(single_tree.keys()))(0):
moms = single_tree[i]
#print(i, moms["bounds"])
level = np.searchsorted(maxind_at_level, i)
spc_bef = ""
if level > 0:
if np.floor((i - 1) / 2.) % 2 or np.floor((i - 1) / 2.) == 0:
if i % 2:
spc_bef = " " + (level - 1) * "| " + "+-"
else:
spc_bef = " " + (level - 1) * "| " + "`-"
else:
if i % 2:
spc_bef = " " + (level - 2) * "| " + " +-"
else:
spc_bef = " " + (level - 2) * "| " + " `-"
spc_aft = (no_levels - level) * " "
bounds_f = '({:>3d}, {:>3d})'.format(*moms['bounds'])
momstr = '{:> 6.2f}, {:> 6.2f}, {:>5.2f}, {:> 3.2f}'.format(
moms['z'], moms['v'], moms['width'], moms['skew'])
if moms['ldr'] != -99:
momstr += ", {:> 5.1f}, {:> 5.1f}".format(moms['ldr'],
moms['ldrmax'])
momstr += ", {:> 5.1f}, {:> 5.1f}".format(moms['thres'],
moms['prominence'])
txt = "{}{:>2d} {}{} | {}".format(spc_bef, i, bounds_f, spc_aft,
momstr)
lines.append(txt)
return '\n'.join(lines)
def __init__(self, _time, _range, *_vel):
# build xarray dataset
super().__init__()
# set metadata
if _time is not None:
self.attrs['ts_unit'], self.attrs['ts_unit_long'] = 'sec', 'Unix time, seconds since Jan 1. 1979'
self.attrs['dt_unit'], self.attrs['dt_unit_long'] = 'date', 'Datetime format'
self.coords['ts'] = _time
self.coords['dt'] = [h.ts_to_dt(ts) for ts in _time]
if _range is not None:
self.attrs['rg_unit'], self.attrs['rg_unit_long'] = 'm', 'Meter'
self.coords['rg'] = _range
# use cloudnet time and range resolution as default
if len(_vel) > 0: self.coords['vel'] = _vel[0]
if len(_vel) > 1: self.coords.update({f'vel_{ic + 1}': _vel[ic] for ic in range(1, len(_vel))})
def plot_time_spectrogram(ZSpec, rg, **font_settings):
rg0 = np.float64(rg)
Z = ZSpec.sel(rg=slice(rg0, rg0 + 40.0),
ts=slice(ZSpec.ts[0], ZSpec.ts[-1]))
z_var = Z.values.copy()
#z_var = np.ma.masked_less_equal(Z.values, 0.0)
z_var = np.squeeze(z_var)
print(z_var.shape)
y_var = np.linspace(0, 6 * 256, num=6 * 256)
z_var = z_var.reshape((-1, 6 * 256), order='F')
fig, ax = plt.subplots(1, figsize=_FIG_SIZE)
surf = ax.pcolormesh([h.ts_to_dt(ts) for ts in Z.ts],
y_var,
z_var[:, :].T,
vmin=dBZ_lim[0],
vmax=dBZ_lim[1],
cmap='jet')
ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter('%H:%M:%S'))
ax.set_xlabel(xlabel=r"time [UTC]")
ax.grid()
# Add a color bar which maps values to colors.
cbar = fig.colorbar(
surf,
fraction=0.2,
shrink=1.,
pad=0.01,
orientation='vertical',
)
cbar.ax.tick_params(axis='both',
which='major',
labelsize=12,
width=2,
length=4)
cbar.ax.tick_params(axis='both', which='minor', width=2, length=3)
cbar.ax.minorticks_on()
cbar.ax.set_ylabel("signal normalized", **{})
plt.tight_layout()
def t_r(f, time_interval, *further_intervals):
"""function that converts the trace netCDF to the data container
"""
logger.debug("filename at reader {}".format(f))
with netCDF4.Dataset(f, 'r') as ncD:
times = ncD.variables[paraminfo['time_variable']][:].astype(
np.float64)
timeconverter, _ = h.get_converter_array(
paraminfo['time_conversion'], ncD=ncD)
ts = timeconverter(times)
#print('timestamps ', ts[:5])
# setup slice to load base on time_interval
it_b = h.argnearest(ts, h.dt_to_ts(time_interval[0]))
if len(time_interval) == 2:
it_e = h.argnearest(ts, h.dt_to_ts(time_interval[1]))
if ts[it_e] < h.dt_to_ts(
time_interval[0]) - 3 * np.median(np.diff(ts)):
logger.warning(
'last profile of file {}\n at {} too far from {}'.
format(f, h.ts_to_dt(ts[it_e]), time_interval[0]))
return None
it_e = it_e + 1 if not it_e == ts.shape[0] - 1 else None
slicer = [slice(it_b, it_e)]
else:
slicer = [slice(it_b, it_b + 1)]
print(slicer)
range_interval = further_intervals[0]
ranges = ncD.variables[paraminfo['range_variable']]
logger.debug('loader range conversion {}'.format(
paraminfo['range_conversion']))
rangeconverter, _ = h.get_converter_array(
paraminfo['range_conversion'], altitude=paraminfo['altitude'])
ir_b = h.argnearest(rangeconverter(ranges[:]), range_interval[0])
if len(range_interval) == 2:
if not range_interval[1] == 'max':
ir_e = h.argnearest(rangeconverter(ranges[:]),
range_interval[1])
ir_e = ir_e + 1 if not ir_e == ranges.shape[0] - 1 else None
else:
ir_e = None
slicer.append(slice(ir_b, ir_e))
else:
slicer.append(slice(ir_b, ir_b + 1))
varconverter, maskconverter = h.get_converter_array(
paraminfo['var_conversion'])
its = np.arange(ts.shape[0])[tuple(slicer)[0]]
irs = np.arange(ranges.shape[0])[tuple(slicer)[1]]
var = np.empty((its.shape[0], irs.shape[0]))
mask = np.empty((its.shape[0], irs.shape[0]))
mask[:] = False
var = ncD.variables[paraminfo['variable_name']][
tuple(slicer)[0], tuple(slicer)[1], :]
data = {}
data['dimlabel'] = ['time', 'range', 'cat']
data["filename"] = f
data["paraminfo"] = paraminfo
data['ts'] = ts[tuple(slicer)[0]]
data['system'] = paraminfo['system']
data['name'] = paraminfo['paramkey']
data['colormap'] = paraminfo['colormap']
if 'meta' in paraminfo:
data['meta'] = NcReader.get_meta_from_nc(
ncD, paraminfo['meta'], paraminfo['variable_name'])
variable = ncD.variables[paraminfo['variable_name']]
var_definition = ast.literal_eval(
variable.getncattr(paraminfo['identifier_var_def']))
if var_definition[1] == "forrest":
var_definition[1] = "forest"
data['var_definition'] = var_definition
data['rg'] = rangeconverter(ranges[tuple(slicer)[1]])
data['rg_unit'] = NcReader.get_var_attr_from_nc(
"identifier_rg_unit", paraminfo, ranges)
logger.debug('shapes {} {} {}'.format(ts.shape, ranges.shape,
var.shape))
data['var_unit'] = NcReader.get_var_attr_from_nc(
"identifier_var_unit", paraminfo, var)
data['var_lims'] = [float(e) for e in \
NcReader.get_var_attr_from_nc("identifier_var_lims",
paraminfo, var)]
data['var'] = varconverter(var)
data['mask'] = maskconverter(mask)
return data
def plot_gn_2d(airmass_source,
plottop=12000,
xright=7000,
xlbl_time='%H:%M',
norm=None):
#f, parameter, dt_list, dsp, config, savepath, config_dict, model):
time_list = airmass_source['ts']
dt_list = [h.ts_to_dt(time) for time in time_list]
height_list = airmass_source['rg'] / 1000.
no_plots = len(dt_list)
if no_plots > 9:
xsize = 15
elif no_plots < 8:
xsize = no_plots * 1.8
else:
xsize = 12
if norm:
airmass_source['var'][:, :, :] = airmass_source['var'][:, :, :].copy(
) / norm
dsp_text = 'Norm. residence time'
else:
dsp_text = 'acc. residence time'
# fig, axes = plt.subplots(1, no_plots, sharex=True, sharey=True,
# figsize=(xsize, 6))
axes = []
fig = plt.figure(constrained_layout=False, figsize=(xsize, 6))
gs1 = fig.add_gridspec(nrows=9, ncols=no_plots, wspace=0.0)
for i in range(no_plots):
axes.append(fig.add_subplot(gs1[1:, i]))
colors = [
(0.65098039215686276, 0.84705882352941175, 0.32941176470588235, 1.0),
(1.0, 0.85098039215686272, 0.18431372549019609, 1.0),
(0.89803921568627454, 0.7686274509803922, 0.58039215686274515, 1.0),
(0.40000000000000002, 0.76078431372549016, 0.6470588235294118, 1.0),
(0.9882352941176471, 0.55294117647058827, 0.3843137254901961, 1.0),
(0.55294117647058827, 0.62745098039215685, 0.79607843137254897, 1.0),
(0.70196078431372544, 0.70196078431372544, 0.70196078431372544, 1.0)
]
for it, dt in enumerate(dt_list):
occ_height = airmass_source['var'][it, :, :]
occ_left = np.cumsum(occ_height, axis=1)
geo_names = airmass_source['var_definition']
l = []
for i in range(len(geo_names)):
if i == 0:
l.append(axes[it].barh(height_list,
occ_height[:, 0].T,
left=0,
align='center',
height=0.3,
color=colors[0],
edgecolor='none'))
else:
l.append(axes[it].barh(height_list,
occ_height[:, i].T,
left=occ_left[:, i - 1].T,
align='center',
height=0.3,
color=colors[i],
edgecolor='none'))
#axes[it].set_ylim([0, 12])
axes[it].set_ylim([0, plottop / 1000.])
axes[it].tick_params(axis='y',
which='major',
labelsize=14,
width=1.5,
length=3)
axes[it].tick_params(axis='both', which='minor', width=1, length=2)
axes[it].tick_params(axis='both',
which='both',
right=True,
top=True,
direction='in')
axes[it].yaxis.set_minor_locator(
matplotlib.ticker.MultipleLocator(1.0))
axes[it].set_xlabel(dt.strftime(xlbl_time), fontsize=14)
axes[it].set_xlim([0, xright])
if xright < 20000:
axes[it].xaxis.set_major_locator(
matplotlib.ticker.MultipleLocator(3000))
else:
# axes[it].xaxis.set_major_locator(matplotlib.ticker.FixedLocator([0, xright/2.]))
axes[it].xaxis.set_major_locator(
matplotlib.ticker.MaxNLocator(nbins=1))
if norm:
axes[it].xaxis.set_major_locator(matplotlib.ticker.AutoLocator())
axes[it].xaxis.set_minor_locator(matplotlib.ticker.AutoMinorLocator())
axes[it].tick_params(axis='x', labeltop=False, labelbottom=False)
axes[it].tick_params(axis='y', labelleft=False)
axes[0].set_ylabel("Height [km]", fontweight='semibold', fontsize=15)
axes[0].tick_params(axis='y', labelleft=True)
axes[-1].tick_params(axis='x',
labeltop=True,
labelbottom=False,
labelsize=13)
plt.suptitle("{} {} {}".format(dt.strftime("%Y%m%d"),
airmass_source['paraminfo']['location'],
airmass_source['name']),
fontweight='semibold',
fontsize=15)
axes[-1].annotate("Endpoint: {:.1f} {:.1f} ".format(
airmass_source['paraminfo']['coordinates'][1],
airmass_source['paraminfo']['coordinates'][0]),
xy=(.88, 0.92),
xycoords='figure fraction',
horizontalalignment='center',
verticalalignment='bottom',
fontsize=12)
axes[0].annotate('Time UTC',
xy=(.5, .02),
xycoords='figure fraction',
horizontalalignment='center',
verticalalignment='bottom',
fontsize=15,
fontweight='semibold')
axes[-1].annotate(dsp_text,
xy=(.87, 0.845),
xycoords='figure fraction',
horizontalalignment='center',
verticalalignment='bottom',
fontsize=13,
fontweight='semibold')
fig.legend(
l,
list(geo_names.values()),
loc='upper left',
#bbox_to_anchor=(0.01, 0.952),
bbox_to_anchor=(0.07, 0.94),
columnspacing=1,
ncol=4,
fontsize=14,
framealpha=0.8)
return fig, axes
def plot_ls_2d(airmass_source,
plottop=12000,
xright=7000,
xlbl_time='%H:%M',
norm=None):
#f, parameter, dt_list, dsp, config, savepath, config_dict, model):
time_list = airmass_source['ts']
dt_list = [h.ts_to_dt(time) for time in time_list]
height_list = airmass_source['rg'] / 1000.
no_plots = len(dt_list)
if no_plots > 9:
xsize = 15
elif no_plots < 8:
xsize = no_plots * 1.8
else:
xsize = 12
if norm:
airmass_source['var'][:, :, :] = airmass_source['var'][:, :, :].copy(
) / norm
dsp_text = 'Norm. residence time'
else:
dsp_text = 'acc. residence time'
axes = []
fig = plt.figure(constrained_layout=False, figsize=(xsize, 6))
gs1 = fig.add_gridspec(nrows=9, ncols=no_plots, wspace=0.0)
for i in range(no_plots):
axes.append(fig.add_subplot(gs1[1:, i]))
#fig, axes = plt.subplots(1, no_plots, sharex=True, sharey=True, figsize=(9, 6))
ls_colors = [
'lightskyblue', 'darkgreen', 'khaki', 'palegreen', 'red', 'gray', 'tan'
]
ls_colors = [
'lightskyblue', 'seagreen', 'khaki', '#6edd6e', 'darkmagenta', 'gray',
'tan'
]
for it, dt in enumerate(dt_list):
occ_height = airmass_source['var'][it, :, :]
occ_left = np.cumsum(occ_height, axis=1)
categories = airmass_source['var_definition']
l1 = axes[it].barh(height_list,
occ_height[:, 0].T,
align='center',
height=0.35,
color=ls_colors[0],
edgecolor='none')
l2 = axes[it].barh(height_list,
occ_height[:, 1].T,
left=occ_left[:, 0].T,
align='center',
height=0.35,
color=ls_colors[1],
edgecolor='none')
l3 = axes[it].barh(height_list,
occ_height[:, 2].T,
left=occ_left[:, 1].T,
align='center',
height=0.35,
color=ls_colors[2],
edgecolor='none')
l4 = axes[it].barh(height_list,
occ_height[:, 3].T,
left=occ_left[:, 2].T,
align='center',
height=0.35,
color=ls_colors[3],
edgecolor='none')
l5 = axes[it].barh(height_list,
occ_height[:, 4].T,
left=occ_left[:, 3].T,
align='center',
height=0.35,
color=ls_colors[4],
edgecolor='none')
l6 = axes[it].barh(height_list,
occ_height[:, 5].T,
left=occ_left[:, 4].T,
align='center',
height=0.35,
color=ls_colors[5],
edgecolor='none')
l7 = axes[it].barh(height_list,
occ_height[:, 6].T,
left=occ_left[:, 5].T,
align='center',
height=0.35,
color=ls_colors[6],
edgecolor='none')
#axes[it].set_ylim([0, 12])
axes[it].set_ylim([0, plottop / 1000.])
axes[it].tick_params(axis='y',
which='major',
labelsize=14,
width=1.5,
length=3)
axes[it].tick_params(axis='both', which='minor', width=1, length=2)
axes[it].tick_params(axis='both',
which='both',
right=True,
top=True,
direction='in')
axes[it].yaxis.set_minor_locator(
matplotlib.ticker.MultipleLocator(1.0))
axes[it].set_xlabel(dt.strftime(xlbl_time), fontsize=14)
axes[it].set_xlim(right=xright)
if xright < 20000:
axes[it].xaxis.set_major_locator(
matplotlib.ticker.MultipleLocator(3000))
else:
axes[it].xaxis.set_major_locator(
matplotlib.ticker.MaxNLocator(nbins=1))
if norm:
axes[it].xaxis.set_major_locator(matplotlib.ticker.AutoLocator())
axes[it].xaxis.set_minor_locator(matplotlib.ticker.AutoMinorLocator())
axes[it].tick_params(axis='x', labeltop=False, labelbottom=False)
axes[it].tick_params(axis='y', labelleft=False)
axes[0].set_ylabel("Height [km]", fontweight='semibold', fontsize=15)
axes[0].tick_params(axis='y', labelleft=True)
axes[-1].tick_params(axis='x',
labeltop=True,
labelbottom=False,
labelsize=13)
plt.suptitle("{} {} {}".format(dt.strftime("%Y%m%d"),
airmass_source['paraminfo']['location'],
airmass_source['name']),
fontweight='semibold',
fontsize=15)
axes[-1].annotate("Endpoint: {:.1f} {:.1f} ".format(
airmass_source['paraminfo']['coordinates'][1],
airmass_source['paraminfo']['coordinates'][0]),
xy=(.88, 0.92),
xycoords='figure fraction',
horizontalalignment='center',
verticalalignment='bottom',
fontsize=12)
axes[0].annotate('Time UTC',
xy=(.5, .02),
xycoords='figure fraction',
horizontalalignment='center',
verticalalignment='bottom',
fontsize=15,
fontweight='semibold')
axes[-1].annotate(dsp_text,
xy=(.87, 0.845),
xycoords='figure fraction',
horizontalalignment='center',
verticalalignment='bottom',
fontsize=13,
fontweight='semibold')
fig.legend(
(l1, l2, l3, l4, l5, l6, l7),
list(categories.values()),
#bbox_to_anchor=(0.85, 0.952),
bbox_to_anchor=(0.07, 0.94),
loc='upper left',
columnspacing=1,
ncol=4,
fontsize=14,
framealpha=0.8)
#fig.set_tight_layout({'rect': [0, 0, 1, 1], 'pad': 0.1, 'h_pad': 1.5})
#plt.tight_layout(w_pad=0.0002)
return fig, axes
def get_param(campaign_name, system, param):
""" """
app.logger.info("got request for {} {} {}".format(campaign_name, system,
param))
starttime = time.time()
larda = pyLARDA.LARDA().connect(campaign_name, build_lists=False)
app.logger.debug("{:5.3f}s load larda".format(time.time() - starttime))
if "rformat" in request.args and request.args['rformat'] == 'bin':
rformat = 'bin'
elif "rformat" in request.args and request.args['rformat'] == 'msgpack':
rformat = 'msgpack'
else:
rformat = 'json'
intervals = request.args.get('interval').split(',')
time_interval = [h.ts_to_dt(float(t)) for t in intervals[0].split('-')]
further_slices = [[
float(e) if e not in ['max'] else e for e in s.split('-')
] for s in intervals[1:]]
app.logger.warning('request.args {}'.format(dict(request.args)))
app.logger.info("time request {}".format(time_interval))
starttime = time.time()
data_container = larda.read(system, param, time_interval, *further_slices,
**dict(request.args))
app.logger.debug("{:5.3f}s read data".format(time.time() - starttime))
starttime = time.time()
for k in ['ts', 'rg', 'vel', 'var', 'mask']:
if k in data_container and hasattr(data_container[k], 'tolist'):
if data_container[k].dtype is not np.dtype('object'):
data_container[k][~np.isfinite(data_container[k])] = 0
data_container[k] = data_container[k].tolist()
app.logger.debug("{:5.3f}s convert data".format(time.time() - starttime))
#import io
#test_datacont = {**data_container}
#for k in ['ts', 'rg', 'var', 'mask']:
# if k in test_datacont:
# print(k,type(test_datacont[k]))
# start = time.time()
# test_datacont[k][~np.isfinite(test_datacont[k])] = 0
# memfile = io.BytesIO()
# np.save(memfile, test_datacont[k])
# memfile.seek(0)
# test_datacont[k] = memfile.read().decode('latin-1')
#resp = Response(json.dumps(test_datacont), status=200, mimetype='application/json')
starttime = time.time()
if rformat == 'bin':
resp = Response(cbor.dumps(data_container),
status=200,
mimetype='application/cbor')
elif rformat == 'msgpack':
resp = Response(msgpack.packb(data_container),
status=200,
mimetype='application/msgpack')
elif rformat == 'json':
resp = Response(json.dumps(data_container),
status=200,
mimetype='application/json')
app.logger.debug("{:5.3f}s dumps {}".format(time.time() - starttime,
rformat))
#for some reason the manual response is faster...
#return jsonify(data_container)
return resp
def pt_ret(f, time_interval, *further_intervals):
"""function that converts the peakTree netCDF to the data container
"""
logger.debug("filename at reader {}".format(f))
with netCDF4.Dataset(f, 'r') as ncD:
times = ncD.variables[paraminfo['time_variable']][:].astype(np.float64)
if 'time_millisec_variable' in paraminfo.keys() and \
paraminfo['time_millisec_variable'] in ncD.variables:
subsec = ncD.variables[paraminfo['time_millisec_variable']][:]/1.0e3
times += subsec
if 'time_microsec_variable' in paraminfo.keys() and \
paraminfo['time_microsec_variable'] in ncD.variables:
subsec = ncD.variables[paraminfo['time_microsec_variable']][:]/1.0e6
times += subsec
timeconverter, _ = h.get_converter_array(
paraminfo['time_conversion'], ncD=ncD)
ts = timeconverter(times)
#print('timestamps ', ts[:5])
# setup slice to load base on time_interval
it_b = h.argnearest(ts, h.dt_to_ts(time_interval[0]))
if len(time_interval) == 2:
it_e = h.argnearest(ts, h.dt_to_ts(time_interval[1]))
if ts[it_e] < h.dt_to_ts(time_interval[0])-3*np.median(np.diff(ts)):
logger.warning(
'last profile of file {}\n at {} too far from {}'.format(
f, h.ts_to_dt(ts[it_e]), time_interval[0]))
return None
it_e = it_e+1 if not it_e == ts.shape[0]-1 else None
slicer = [slice(it_b, it_e)]
else:
slicer = [slice(it_b, it_b+1)]
print(slicer)
if paraminfo['ncreader'] == 'peakTree':
range_tg = True
range_interval = further_intervals[0]
ranges = ncD.variables[paraminfo['range_variable']]
logger.debug('loader range conversion {}'.format(paraminfo['range_conversion']))
rangeconverter, _ = h.get_converter_array(
paraminfo['range_conversion'],
altitude=paraminfo['altitude'])
ir_b = h.argnearest(rangeconverter(ranges[:]), range_interval[0])
if len(range_interval) == 2:
if not range_interval[1] == 'max':
ir_e = h.argnearest(rangeconverter(ranges[:]), range_interval[1])
ir_e = ir_e+1 if not ir_e == ranges.shape[0]-1 else None
else:
ir_e = None
slicer.append(slice(ir_b, ir_e))
else:
slicer.append(slice(ir_b, ir_b+1))
varconverter, maskconverter = h.get_converter_array(
paraminfo['var_conversion'])
its = np.arange(ts.shape[0])[tuple(slicer)[0]]
irs = np.arange(ranges.shape[0])[tuple(slicer)[1]]
var = np.empty((its.shape[0], irs.shape[0]), dtype=object)
mask = np.empty((its.shape[0], irs.shape[0]), dtype=bool)
mask[:] = True
param_list = [
ncD.variables['parent'][tuple(slicer)[0],tuple(slicer)[1],:], #0
ncD.variables['Z'][tuple(slicer)[0],tuple(slicer)[1],:], #1
ncD.variables['v'][tuple(slicer)[0],tuple(slicer)[1],:], #2
ncD.variables['width'][tuple(slicer)[0],tuple(slicer)[1],:], #3
ncD.variables['skew'][tuple(slicer)[0],tuple(slicer)[1],:], #4
ncD.variables['threshold'][tuple(slicer)[0],tuple(slicer)[1],:], #5
ncD.variables['prominence'][tuple(slicer)[0],tuple(slicer)[1],:], #6
ncD.variables['bound_l'][tuple(slicer)[0],tuple(slicer)[1],:], #7
ncD.variables['bound_r'][tuple(slicer)[0],tuple(slicer)[1],:] #8
]
if 'LDR' in ncD.variables.keys():
ldr_avail = True
param_list.append(ncD.variables['LDR'][tuple(slicer)[0],tuple(slicer)[1],:]) #9
param_list.append(ncD.variables['ldrmax'][tuple(slicer)[0],tuple(slicer)[1],:]) #10
else:
ldr_avail = False
data = np.stack(tuple(param_list), axis=3)
print(data.shape)
if fastbuilder:
var, mask = peakTree_fastbuilder.array_to_tree_c(data.astype(float), ldr_avail)
else:
var, mask = array_to_tree_py(data, ldr_avail)
data = {}
data['dimlabel'] = ['time', 'range', 'dict']
data["filename"] = f
data["paraminfo"] = paraminfo
data['ts'] = ts[tuple(slicer)[0]]
data['system'] = paraminfo['system']
data['name'] = paraminfo['paramkey']
data['colormap'] = paraminfo['colormap']
data['rg'] = rangeconverter(ranges[tuple(slicer)[1]])
data['rg_unit'] = NcReader.get_var_attr_from_nc("identifier_rg_unit",
paraminfo, ranges)
logger.debug('shapes {} {} {}'.format(ts.shape, ranges.shape, var.shape))
logger.debug('shapes {} {}'.format(ts.shape, var.shape))
data['var_unit'] = NcReader.get_var_attr_from_nc("identifier_var_unit",
paraminfo, var)
data['var_lims'] = [float(e) for e in \
NcReader.get_var_attr_from_nc("identifier_var_lims",
paraminfo, var)]
data['var'] = varconverter(var)
data['mask'] = maskconverter(mask)
return data
def export_spectra2nc(data, larda_git_path='', system='', path='', **kwargs):
"""
This routine generates an hourly NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'.
Args:
data (dict): dictionary of larda containers
system (string): name of the radar system
path (string): path where the NetCDF file is stored
"""
hour_bias = kwargs['hour_bias'] if 'hour_bias' in kwargs else 0
dt_start = h.ts_to_dt(data['time'][0])
dt_end = h.ts_to_dt(data['time'][-1])
ds_name = path + f'{dt_start:%Y%m%d-%H%M}-{dt_end:%H%M}_{system}_spectra.nc'
repo = git.Repo(larda_git_path)
sha = repo.head.object.hexsha
print('open file: ', ds_name)
with netCDF4.Dataset(ds_name, "w", format="NETCDF4") as ds:
ds.git_description = f'GIT commit ID {sha}'
ds.description = f'{system} calibrated Doppler spectra'
ds.history = 'Created ' + time.ctime(time.time())
ds.system = system
ds.location = data['location']
ds._FillValue = data['fill_value']
ds.createDimension(
'chirp', data['chirps']) # add variable number of chirps later
ds.createDimension('time', data['time'].size)
ds.createDimension('range', data['range'].size)
ds.createDimension('velocity', data['velocity'].shape[1])
nc_add_variable(ds,
val=data['coordinates'][0],
dimension=(),
var_name='latitude',
type=np.float32,
long_name='GPS latitude',
unit='deg')
nc_add_variable(ds,
val=data['coordinates'][1],
dimension=(),
var_name='longitude',
type=np.float32,
long_name='GPS longitude',
unit='deg')
nc_add_variable(ds,
val=data['altitude'],
dimension=(),
var_name='altitude',
type=np.float32,
long_name='altitude, i.e. height above sea level',
unit='m')
nc_add_variable(
ds,
val=data['time'],
dimension=('time', ),
var_name='time',
type=np.float64,
long_name='Unix Time - seconds since 01.01.1970 00:00 UTC',
unit='sec')
nc_add_variable(ds,
val=data['rg_offsets'],
dimension=('chirp', ),
var_name='rg_offsets',
type=np.float32,
long_name='Range Indices for next chirp sequence.',
unit='-')
nc_add_variable(ds,
val=data['range'],
dimension=('range', ),
var_name='range',
type=np.float32,
long_name='range',
unit='m')
nc_add_variable(ds,
val=data['velocity'],
dimension=(
'chirp',
'velocity',
),
var_name=f'velocity_vectors',
type=np.float32,
long_name='velocity vectors for each chirp',
unit='m s-1')
nc_add_variable(ds,
val=data['nyquist_velocity'],
dimension=('chirp', ),
var_name='nyquist_velocity',
type=np.float32,
long_name='Unambiguous Doppler velocity (+/-)',
unit='m s-1')
nc_add_variable(
ds,
val=data['doppler_spectrum'],
dimension=('time', 'range', 'velocity'),
var_name='doppler_spectrum',
type=np.float32,
long_name=
'Doppler spectrum, if dual polarization radar: doppler_spectrum = vertical + horizontal polarization',
unit='mm6 m-3 (m s-1)-1')
try:
nc_add_variable(ds,
val=data['covariance_spectrum_re'],
dimension=('time', 'range', 'velocity'),
var_name='covariance_spectrum_re',
type=np.float32,
long_name='Real part of covariance spectrum',
unit='mm6 m-3')
except KeyError:
print('skip writing real part of covariance spectrum')
try:
nc_add_variable(
ds,
val=data['covariance_spectrum_im'],
dimension=('time', 'range', 'velocity'),
var_name='covariance_spectrum_im',
type=np.float32,
long_name='Imaginary part of covariance spectrum',
unit='mm6 m-3')
except KeyError:
print('skip writing imaginary part of covariance spectrum')
try:
nc_add_variable(
ds,
val=data['sensitivity_limit'],
dimension=('time', 'range'),
var_name='sensitivity_limit',
type=np.float32,
long_name=
'Sensitivity limit, if dual polarization radar: sensitivity_limit = vertical + horizontal polarization',
unit='mm6 m-3')
except KeyError:
print('skip writing sensitivity limit')
try:
nc_add_variable(
ds,
val=data['doppler_spectrum_h'],
dimension=('time', 'range', 'velocity'),
var_name='doppler_spectrum_h',
type=np.float32,
long_name='Doppler spectrum, horizontal polarization only',
unit='mm6 m-3 ')
nc_add_variable(
ds,
val=data['sensitivity_limit_h'],
dimension=('time', 'range'),
var_name='sensitivity_limit_h',
type=np.float32,
long_name='Sensitivity limit for horizontal polarization',
unit='mm6 m-3 (m s-1)-1')
except KeyError:
print('Skip writing horizontal polarization.')
return 0
def rpg_radar2nc_old(data, path, **kwargs):
"""
This routine generates a daily NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'.
Args:
data (dict): dictionary of larda containers
path (string): path where the NetCDF file is stored
"""
import time
no_chirps = 3
if 'time_frame' in kwargs:
ds_name = path + kwargs['time_frame'] + '_LIMRAD94.nc'.format(
h.ts_to_dt(data['Ze']['ts'][0]).strftime("%Y%m%d"),
kwargs['time_frame'])
else:
ds_name = path + '{}_000000-240000_LIMRAD94.nc'.format(
h.ts_to_dt(data['Ze']['ts'][0]).strftime("%Y%m%d"))
with netCDF4.Dataset(ds_name, "w", format="NETCDF4") as ds:
# ds.commit_id = subprocess.check_output(["git", "describe", "--always"]) .rstrip()
ds.description = 'Concatenated data files of LIMRAD 94GHz - FMCW Radar, used as input for Cloudnet processing, ' \
'filters applied: ghos-echo, despeckle, use only main peak'
ds.history = 'Created ' + time.ctime(time.time())
ds.source = data['Ze']['paraminfo']['location']
ds._FillValue = data['Ze']['paraminfo']['fill_value']
ds.createDimension('chirp',
no_chirps) # add variable number of chirps later
ds.createDimension('time', data['Ze']['ts'].size)
ds.createDimension('range', data['Ze']['rg'].size)
# coordinates
nc_add_variable(ds,
val=data['Ze']['paraminfo']['coordinates'][0],
dimension=(),
var_name='latitude',
type=np.float32,
long_name='GPS latitude',
unit='deg')
nc_add_variable(ds,
val=data['Ze']['paraminfo']['coordinates'][1],
dimension=(),
var_name='longitude',
type=np.float32,
long_name='GPS longitude',
unit='deg')
# time and range variable
# convert to time since 20010101
ts = np.subtract(data['Ze']['ts'],
datetime.datetime(2001, 1, 1, 0, 0, 0).timestamp())
nc_add_variable(ds,
val=ts,
dimension=('time', ),
var_name='time',
type=np.float64,
long_name='Seconds since 01.01.2001 00:00 UTC',
unit='sec')
nc_add_variable(ds,
val=data['Ze']['rg'],
dimension=('range', ),
var_name='range',
type=np.float32,
long_name='range',
unit='m')
# 2D variables
nc_add_variable(ds,
val=data['Ze']['var'].T,
dimension=(
'range',
'time',
),
var_name='Ze',
type=np.float32,
long_name='Equivalent radar reflectivity factor',
unit='mm^6/m^3')
nc_add_variable(ds,
val=data['VEL']['var'].T,
dimension=(
'range',
'time',
),
var_name='vm',
type=np.float32,
long_name='Mean Doppler velocity',
unit='m/s')
nc_add_variable(ds,
val=data['sw']['var'].T,
dimension=(
'range',
'time',
),
var_name='sigma',
type=np.float32,
long_name='Spectrum width',
unit='m/s')
nc_add_variable(ds,
val=data['ldr']['var'].T,
dimension=(
'range',
'time',
),
var_name='ldr',
type=np.float32,
long_name='Slanted linear depolarization ratio',
unit='dB')
nc_add_variable(ds,
val=data['kurt']['var'].T,
dimension=(
'range',
'time',
),
var_name='kurt',
type=np.float32,
long_name='Kurtosis',
unit='[linear]')
nc_add_variable(ds,
val=data['skew']['var'].T,
dimension=(
'range',
'time',
),
var_name='Skew',
type=np.float32,
long_name='Skewness',
unit='[linear]')
nc_add_variable(ds,
val=data['DiffAtt']['var'].T,
dimension=(
'range',
'time',
),
var_name='DiffAtt',
type=np.float32,
long_name='Differential attenuation',
unit='dB/km')
# 1D variables
nc_add_variable(ds,
val=data['bt']['var'],
dimension=('time', ),
var_name='bt',
type=np.float32,
long_name='Direct detection brightness temperature',
unit='K')
nc_add_variable(ds,
val=data['LWP']['var'],
dimension=('time', ),
var_name='lwp',
type=np.float32,
long_name='Liquid water path',
unit='g/m^2')
nc_add_variable(ds,
val=data['rr']['var'],
dimension=('time', ),
var_name='rain',
type=np.float32,
long_name='Rain rate from weather station',
unit='mm/h')
nc_add_variable(ds,
val=data['SurfRelHum']['var'],
dimension=('time', ),
var_name='SurfRelHum',
type=np.float32,
long_name='Relative humidity from weather station',
unit='%')
# chirp dependent variables
nc_add_variable(ds,
val=data['MaxVel']['var'][0],
dimension=('chirp', ),
var_name='DoppMax',
type=np.float32,
long_name='Unambiguous Doppler velocity (+/-)',
unit='m/s')
range_offsets = np.ones(no_chirps, dtype=np.uint)
for iC in range(no_chirps - 1):
range_offsets[iC + 1] = range_offsets[iC] + data[
'C' + str(iC + 1) + 'Range']['var'][0].shape
nc_add_variable(
ds,
val=range_offsets,
dimension=('chirp', ),
var_name='range_offsets',
type=np.uint,
long_name=
'chirp sequences start index array in altitude layer array',
unit='[-]')
print('save calibrated to :: ', ds_name)
return 0
def rpg_radar2nc(data, path, larda_git_path, **kwargs):
"""
This routine generates a daily NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'.
Args:
data (dict): dictionary of larda containers
path (string): path where the NetCDF file is stored
"""
dt_start = h.ts_to_dt(data['Ze']['ts'][0])
h.make_dir(path)
site_name = kwargs['site'] if 'site' in kwargs else 'no-site'
hour_bias = kwargs['hour_bias'] if 'hour_bias' in kwargs else 0
cn_version = kwargs['version'] if 'version' in kwargs else 'pyhon'
ds_name = f'{path}/{h.ts_to_dt(data["Ze"]["ts"][0]):%Y%m%d}-{site_name}-limrad94.nc'
ncvers = '4'
repo = git.Repo(larda_git_path)
sha = repo.head.object.hexsha
with netCDF4.Dataset(ds_name, 'w', format=f'NETCDF{ncvers}') as ds:
ds.Convention = 'CF-1.0'
ds.location = data['Ze']['paraminfo']['location']
ds.system = data['Ze']['paraminfo']['system']
ds.version = f'Variable names and dimensions prepared for Cloudnet {kwargs["version"]} version'
ds.title = 'LIMRAD94 (SLDR) Doppler Cloud Radar, calibrated Input for Cloudnet'
ds.institution = 'Leipzig Institute for Meteorology (LIM), Leipzig, Germany'
ds.source = '94 GHz Cloud Radar LIMRAD94\nRadar type: Frequency Modulated Continuous Wave,\nTransmitter power 1.5 W typical (solid state ' \
'amplifier)\nAntenna Type: Bi-static Cassegrain with 500 mm aperture\nBeam width: 0.48deg FWHM'
ds.reference = 'W Band Cloud Radar LIMRAD94\nDocumentation and User Manual provided by manufacturer RPG Radiometer Physics GmbH\n' \
'Information about system also available at https://www.radiometer-physics.de/'
ds.calibrations = f'remove Precip. ghost: {kwargs["ghost_echo_1"]}\n, remove curtain ghost: {kwargs["ghost_echo_2"]}\n' \
f'despeckle: {kwargs["despeckle"]}\n, number of standard deviations above noise: {kwargs["NF"]}\n'
ds.git_description = f'GIT commit ID {sha}'
ds.description = 'Concatenated data files of LIMRAD 94GHz - FMCW Radar, used as input for Cloudnet processing, ' \
'filters applied: ghos-echo, despeckle, use only main peak'
ds.history = 'Created ' + time.ctime(time.time())
ds._FillValue = data['Ze']['paraminfo']['fill_value']
ds.day = dt_start.day
ds.month = dt_start.month
ds.year = dt_start.year
# ds.commit_id = subprocess.check_output(["git", "describe", "--always"]) .rstrip()
ds.history = 'Created ' + time.ctime(time.time(
)) + '\nfilters applied: ghost-echo, despeckle, main peak only'
Ze_str = 'Zh' if cn_version == 'python' else 'Ze'
vel_str = 'v' if cn_version == 'python' else 'vm'
width_str = 'width' if cn_version == 'python' else 'sigma'
dim_tupel = ('time', 'range') if cn_version == 'python' else ('range',
'time')
n_chirps = len(data['no_av'])
ds.createDimension('chirp', n_chirps)
ds.createDimension('time', data['Ze']['ts'].size)
ds.createDimension('range', data['Ze']['rg'].size)
if cn_version == 'matlab':
for ivar in ['Ze', 'VEL', 'sw', 'ldr', 'kurt', 'skew']:
data[ivar]['var'] = data[ivar]['var'].T
# coordinates
nc_add_variable(ds,
val=94.0,
dimension=(),
var_name='frequency',
type=np.float32,
long_name='Radar frequency',
units='GHz')
nc_add_variable(ds,
val=256,
dimension=(),
var_name='Numfft',
type=np.float32,
long_name='Number of points in FFT',
units='')
nc_add_variable(
ds,
val=np.mean(data['MaxVel']['var']),
dimension=(),
var_name='NyquistVelocity',
type=np.float32,
long_name=
'Mean (over all chirps) Unambiguous Doppler velocity (+/-)',
units='m s-1')
nc_add_variable(ds,
val=data['Ze']['paraminfo']['altitude'],
dimension=(),
var_name='altitude',
type=np.float32,
long_name='Height of instrument above mean sea level',
units='m')
nc_add_variable(ds,
val=data['Ze']['paraminfo']['coordinates'][0],
dimension=(),
var_name='latitude',
type=np.float32,
long_name='latitude',
units='degrees_north')
nc_add_variable(ds,
val=data['Ze']['paraminfo']['coordinates'][1],
dimension=(),
var_name='longitude',
type=np.float32,
long_name='longitude',
units='degrees_east')
if 'version' in kwargs and cn_version == 'python':
nc_add_variable(ds,
val=data['no_av'],
dimension=('chirp', ),
var_name='NumSpectraAveraged',
type=np.float32,
long_name='Number of spectral averages',
units='')
# time and range variable
# convert to time since midnight
if cn_version == 'python':
ts = np.subtract(
data['Ze']['ts'],
datetime.datetime(dt_start.year, dt_start.month, dt_start.day,
hour_bias, 0, 0).timestamp())
ts_str = 'Decimal hours from midnight UTC to the middle of each day'
ts_unit = f'hours since {dt_start:%Y-%m-%d} 00:00:00 +00:00 (UTC)'
rg = data['Ze']['rg'] / 1000.0
elif cn_version == 'matlab':
ts = np.subtract(
data['Ze']['ts'],
datetime.datetime(2001, 1, 1, hour_bias, 0, 0).timestamp())
ts_str = 'Seconds since 1st January 2001 00:00 UTC'
ts_unit = 'sec'
rg = data['Ze']['rg']
else:
raise ValueError(
'Wrong version selected! version to "matlab" or "python"!')
nc_add_variable(ds,
val=ts,
dimension=('time', ),
var_name='time',
type=np.float64,
long_name=ts_str,
units=ts_unit)
nc_add_variable(
ds,
val=rg,
dimension=('range', ),
var_name='range',
type=np.float32,
long_name='Range from antenna to the centre of each range gate',
units='km')
nc_add_variable(ds,
val=data['Azm']['var'],
dimension=('time', ),
var_name='azimuth',
type=np.float32,
long_name='Azimuth angle from north',
units='degree')
nc_add_variable(
ds,
val=data['Elv']['var'],
dimension=('time', ),
var_name='elevation',
type=np.float32,
long_name='elevation angle. 90 degree is vertical direction.',
units='degree')
# chirp dependent variables
nc_add_variable(ds,
val=data['MaxVel']['var'][0],
dimension=('chirp', ),
var_name='DoppMax',
type=np.float32,
long_name='Unambiguous Doppler velocity (+/-)',
units='m s-1')
# index plus (1 to n) for Matlab indexing
nc_add_variable(
ds,
val=data['rg_offsets'],
dimension=('chirp', ),
var_name='range_offsets',
type=np.uint32,
long_name=
'chirp sequences start index array in altitude layer array',
units='-')
# 1D variables
nc_add_variable(ds,
val=data['bt']['var'],
dimension=('time', ),
var_name='bt',
type=np.float32,
long_name='Direct detection brightness temperature',
units='K')
nc_add_variable(ds,
val=data['LWP']['var'],
dimension=('time', ),
var_name='lwp',
type=np.float32,
long_name='Liquid water path',
units='g m-2')
nc_add_variable(ds,
val=data['rr']['var'],
dimension=('time', ),
var_name='rain',
type=np.float32,
long_name='Rain rate from weather station',
units='mm h-1')
nc_add_variable(ds,
val=data['SurfRelHum']['var'],
dimension=('time', ),
var_name='SurfRelHum',
type=np.float32,
long_name='Relative humidity from weather station',
units='%')
# 2D variables
nc_add_variable(
ds,
val=data['Ze']['var'],
dimension=dim_tupel,
var_name=Ze_str,
type=np.float32,
long_name='Radar reflectivity factor',
units='mm6 m-3',
plot_range=data['Ze']['var_lims'],
plot_scale='linear',
comment=
'Calibrated reflectivity. Calibration convention: in the absence of attenuation, '
'a cloud at 273 K containing one million 100-micron droplets per cubic metre will '
'have a reflectivity of 0 dBZ at all frequencies.')
nc_add_variable(
ds,
val=data['VEL']['var'],
dimension=dim_tupel,
plot_range=data['VEL']['var_lims'],
plot_scale='linear',
var_name=vel_str,
type=np.float32,
long_name='Doppler velocity',
units='m s-1',
unit_html='m s<sup>-1</sup>',
comment=
'This parameter is the radial component of the velocity, with positive velocities are away from the radar.',
folding_velocity=data['MaxVel']['var'].max())
nc_add_variable(
ds,
val=data['sw']['var'],
dimension=dim_tupel,
plot_range=data['sw']['var_lims'],
lot_scale='logarithmic',
var_name=width_str,
type=np.float32,
long_name='Spectral width',
units='m s-1',
unit_html='m s<sup>-1</sup>',
comment=
'This parameter is the standard deviation of the reflectivity-weighted velocities in the radar pulse volume.'
)
nc_add_variable(
ds,
val=data['ldr']['var'],
dimension=dim_tupel,
plot_range=[-30.0, 0.0],
var_name='ldr',
type=np.float32,
long_name='Linear depolarisation ratio',
units='dB',
comment=
'This parameter is the ratio of cross-polar to co-polar reflectivity.'
)
nc_add_variable(ds,
val=data['kurt']['var'],
dimension=dim_tupel,
plot_range=data['kurt']['var_lims'],
var_name='kurt',
type=np.float32,
long_name='Kurtosis',
units='linear')
nc_add_variable(ds,
val=data['skew']['var'],
dimension=dim_tupel,
plot_range=data['skew']['var_lims'],
var_name='Skew',
type=np.float32,
long_name='Skewness',
units='linear')
print('save calibrated to :: ', ds_name)
return 0
def retfunc(f, time_interval, range_interval):
"""function that converts the netCDF to the larda-data-format
"""
logger.debug("filename at reader {}".format(f))
with netCDF4.Dataset(f, 'r') as ncD:
ranges = ncD.variables[paraminfo['range_variable']]
times = ncD.variables[paraminfo['time_variable']][:].astype(
np.float64)
locator_mask = ncD.variables[paraminfo['mask_var']][:].astype(
np.int)
if 'time_millisec_variable' in paraminfo.keys() and \
paraminfo['time_millisec_variable'] in ncD.variables:
subsec = ncD.variables[
paraminfo['time_millisec_variable']][:] / 1.0e3
times += subsec
if 'time_microsec_variable' in paraminfo.keys() and \
paraminfo['time_microsec_variable'] in ncD.variables:
subsec = ncD.variables[
paraminfo['time_microsec_variable']][:] / 1.0e6
times += subsec
if 'base_time_variable' in paraminfo.keys() and \
paraminfo['base_time_variable'] in ncD.variables:
basetime = ncD.variables[
paraminfo['base_time_variable']][:].astype(np.float64)
times += basetime
timeconverter, _ = h.get_converter_array(
paraminfo['time_conversion'], ncD=ncD)
ts = timeconverter(times)
it_b = np.searchsorted(ts,
h.dt_to_ts(time_interval[0]),
side='right')
if len(time_interval) == 2:
it_e = h.argnearest(ts, h.dt_to_ts(time_interval[1]))
if it_b == ts.shape[0]: it_b = it_b - 1
if ts[it_e] < h.dt_to_ts(time_interval[0]) - 3 * np.median(np.diff(ts)) \
or ts[it_b] < h.dt_to_ts(time_interval[0]):
# second condition is to ensure that no timestamp before
# the selected interval is chosen
# (problem with limrad after change of sampling frequency)
logger.warning(
'last profile of file {}\n at {} too far from {}'.
format(f, h.ts_to_dt(ts[it_e]), time_interval[0]))
return None
it_e = it_e + 1 if not it_e == ts.shape[0] - 1 else None
slicer = [slice(it_b, it_e)]
elif it_b == ts.shape[0]:
# only one timestamp is selected
# and the found right one would be beyond the ts range
it_b = h.argnearest(ts, h.dt_to_ts(time_interval[0]))
slicer = [slice(it_b, it_b + 1)]
else:
slicer = [slice(it_b, it_b + 1)]
rangeconverter, _ = h.get_converter_array(
paraminfo['range_conversion'])
varconverter, _ = h.get_converter_array(
paraminfo['var_conversion'])
ir_b = h.argnearest(rangeconverter(ranges[:]), range_interval[0])
if len(range_interval) == 2:
if not range_interval[1] == 'max':
ir_e = h.argnearest(rangeconverter(ranges[:]),
range_interval[1])
ir_e = ir_e + 1 if not ir_e == ranges.shape[0] - 1 else None
else:
ir_e = None
slicer.append(slice(ir_b, ir_e))
else:
slicer.append(slice(ir_b, ir_b + 1))
range_out = rangeconverter(ranges[tuple(slicer)[1]])
cal = getattr(ncD, paraminfo['cal_const'])
var = ncD.variables[paraminfo['variable_name']][:].astype(
np.float64)
var = var[locator_mask]
vel = ncD.variables[paraminfo['vel_variable']][:].astype(
np.float64)
# print('var dict ',ch1var.__dict__)
# print('shapes ', ts.shape, ch1range.shape, ch1var.shape)
# print("time indices ", it_b, it_e)
data = {}
data['dimlabel'] = ['time', 'range', 'vel']
data["filename"] = f
data["paraminfo"] = paraminfo
data['ts'] = ts[tuple(slicer)[0]]
data['rg'] = range_out
data['system'] = paraminfo['system']
data['name'] = paraminfo['paramkey']
data['colormap'] = paraminfo['colormap']
# also experimental: vis_varconverter
if 'plot_varconverter' in paraminfo and paraminfo[
'plot_varconverter'] != 'none':
data['plot_varconverter'] = paraminfo['plot_varconverter']
else:
data['plot_varconverter'] = ''
data['rg_unit'] = get_var_attr_from_nc("identifier_rg_unit",
paraminfo, ranges)
#data['var_unit'] = get_var_attr_from_nc("identifier_var_unit",
# paraminfo, var)
data['var_unit'] = 'dBZ m-1 s'
data['var_lims'] = [float(e) for e in \
get_var_attr_from_nc("identifier_var_lims",
paraminfo, var)]
data['vel'] = vel
if "identifier_fill_value" in paraminfo.keys(
) and not "fill_value" in paraminfo.keys():
fill_value = var.getncattr(paraminfo['identifier_fill_value'])
data['mask'] = (var[tuple(slicer)].data == fill_value)
elif "fill_value" in paraminfo.keys():
fill_value = paraminfo["fill_value"]
data['mask'] = np.isclose(var[tuple(slicer)], fill_value)
elif "mask_var" in paraminfo.keys():
# combine locator mask and mask of infinite values
mask = locator_mask.mask[tuple(slicer)]
data["mask"] = np.logical_or(
~np.isfinite(var[tuple(slicer)].data),
np.repeat(mask[:, :, np.newaxis], len(data['vel']),
axis=2))
else:
data['mask'] = ~np.isfinite(var[tuple(slicer)].data)
if isinstance(times, np.ma.MaskedArray):
var = varconverter(var[tuple(slicer)].data)
else:
var = varconverter(var[tuple(slicer)])
var2 = h.z2lin(var) * h.z2lin(float(
cal[:-3])) * (range_out**2)[np.newaxis, :, np.newaxis]
data['var'] = var2
return data
def generate_weather_file_LIMRAD94(data, path, **kwargs):
"""
This routine generates a daily NetCDF4 file with the weather station measurements from the RPG 94 GHz FMCW radar
'LIMRAD94'.
Args:
data (dict): dictionary of larda containers
path (string): path where the NetCDF file is stored
"""
import time
if 'time_frame' in kwargs:
ds_name = path + kwargs['time_frame'] + \
'_LIMRAD94_weather.nc'.format(h.ts_to_dt(data['SurfWS']['ts'][0]).strftime("%Y%m%d"), kwargs['time_frame'])
else:
ds_name = path + '{}_000000-240000_LIMRAD94_weather.nc'.format(h.ts_to_dt(data['SurfWS']['ts'][0]).strftime("%Y%m%d"))
ds = netCDF4.Dataset(ds_name, "w", format="NETCDF4")
ds.description = 'Concatenated data files of LIMRAD 94GHz - FMCW Radar, Data from the Vaisalla weather station'
ds.history = 'Created ' + time.ctime(time.time())
ds.source = data['SurfWS']['paraminfo']['location']
ds.FillValue = data['SurfWS']['paraminfo']['fill_value']
ds.createDimension('time', data['SurfWS']['ts'].size)
# coordinates
nc_add_variable(ds, val=data['SurfWS']['paraminfo']['coordinates'][0], dimension=(),
var_name='latitude', type=np.float32, long_name='GPS latitude', unit='deg')
nc_add_variable(ds, val=data['SurfWS']['paraminfo']['coordinates'][1], dimension=(),
var_name='longitude', type=np.float32, long_name='GPS longitude', unit='deg')
# time
# convert to time since 20010101
ts = np.subtract(data['SurfWS']['ts'], datetime.datetime(2001, 1, 1, 0, 0, 0).timestamp())
nc_add_variable(ds, val=ts, dimension=('time',),
var_name='time', type=np.float64, long_name='Seconds since 2001-01-01 00:00 UTC', unit='seconds')
# 1D variables
nc_add_variable(ds, val=data['SurfRelHum']['var'], dimension=('time',),
var_name='SurfRelHum', type=np.float32,
long_name='Relative humidity from weather station', unit='%')
nc_add_variable(ds, val=data['SurfTemp']['var'], dimension=('time',),
var_name='SurfTemp', type=np.float32, long_name='Surface temperature from weather station',
unit='K')
nc_add_variable(ds, val=data['SurfPres']['var'], dimension=('time',),
var_name='SurfPres', type=np.float32,
long_name='Surface atmospheric pressure from weather station', unit='hPa')
nc_add_variable(ds, val=data['SurfWS']['var'], dimension=('time',),
var_name='SurfWS', type=np.float32, long_name='Surface wind speed from weather station',
unit='m/s')
nc_add_variable(ds, val=data['SurfWD']['var'], dimension=('time',),
var_name='SurfWD', type=np.float32,
long_name='Surface wind direction from weather station',
unit='deg')
nc_add_variable(ds, val=data['rr']['var'], dimension=('time',),
var_name='Rain', type=np.float32, long_name='Rain rate from weather station', unit='mm/h')
ds.close()
print('save calibrated to :: ', ds_name)
return 0
def rpg_radar2nc_eurec4a(data, path, **kwargs):
"""
This routine generates a daily NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'.
The variables and metadata are designed for the EUREC4A campaign.
Args:
data (dict): dictionary of larda containers
path (string): path where the NetCDF file is stored
"""
dt_start = h.ts_to_dt(data['Ze']['ts'][0])
h.make_dir(path)
site_name = kwargs['site'] if 'site' in kwargs else 'no-site'
cn_version = kwargs['version'] if 'version' in kwargs else 'python'
hc_version = kwargs['heave_corr_version'] if 'heave_corr_version' in kwargs else None
for_aeris = kwargs['for_aeris'] if 'for_aeris' in kwargs else False
dataset_version = 'v1.2'
ds_name = f'{path}/eurec4a_{site_name}_cloudradar_{h.ts_to_dt(data["Ze"]["ts"][0]):%Y%m%d}_{dataset_version}.nc'
ncvers = '4'
with netCDF4.Dataset(ds_name, 'w', format=f'NETCDF{ncvers}') as ds:
ds.Conventions = 'CF-1.8'
ds.title = 'LIMRAD94 (SLDR) Doppler Cloud Radar, calibrated file'
ds.campaign_id = 'EUREC4A'
ds.platform_id = 'Meteor'
ds.instrument_id = 'LIMRAD94'
ds.version_id = dataset_version
ds.location = data['Ze']['paraminfo']['location']
ds.system = data['Ze']['paraminfo']['system']
ds.version = f'Variable names and dimensions prepared for upload to Aeris data center. 1.1: updated metadata; ' \
f'1.2: added hydrometeor_mask'
ds.institution = 'Leipzig Institute for Meteorology (LIM), Leipzig, Germany'
ds.contact = '*****@*****.**'
ds.source = '94 GHz Cloud Radar LIMRAD94\nRadar type: Frequency Modulated Continuous Wave,' \
'\nTransmitter power 1.5 W typical (solid state amplifier)' \
'\nAntenna Type: Bi-static Cassegrain with 500 mm aperture\nBeam width: 0.48deg FWHM'
ds.reference = 'W Band Cloud Radar LIMRAD94' \
'\nDocumentation and User Manual provided by manufacturer RPG Radiometer Physics GmbH\n' \
'Information about system also available at https://www.radiometer-physics.de/'
ds.calibrations = f'remove Precip. ghost: {kwargs["ghost_echo_1"]}\n, ' \
f'remove curtain ghost: {kwargs["ghost_echo_2"]}\n' \
f'despeckle: {kwargs["despeckle"]}\n, ' \
f'number of standard deviations above noise: {kwargs["NF"]}\n' \
f'spectra heave corrected: {kwargs["heave_correction"]}\n' \
f'spectra heave corrected with version: {hc_version}\n' \
f'spectra dealiased: {kwargs["dealiasing"]}'
ds.description = 'Concatenated data files of LIMRAD 94GHz - FMCW Radar, ' \
'filters applied: ghost-echo, despeckle, use only main peak,\n' \
'spectra corrected for heave motion of ship and then dealiased'
ds.history = 'Created ' + time.ctime(time.time())
ds._FillValue = data['Ze']['paraminfo']['fill_value']
ds.day = dt_start.day
ds.month = dt_start.month
ds.year = dt_start.year
Ze_str = 'Zh' if cn_version == 'python' else 'Ze'
vel_str = 'v' if cn_version == 'python' else 'vm'
width_str = 'width' if cn_version == 'python' else 'sigma'
dim_tupel = ('time', 'range') if cn_version == 'python' else ('range', 'time')
n_chirps = len(data['no_av'])
ds.createDimension('chirp', n_chirps)
ds.createDimension('time', data['Ze']['ts'].size)
ds.createDimension('range', data['Ze']['rg'].size)
if cn_version == 'matlab':
for ivar in ['Ze', 'VEL', 'sw', 'ldr', 'kurt', 'skew']:
data[ivar]['var'] = data[ivar]['var'].T
# scalar variables
nc_add_variable(
ds,
val=94.0,
dimension=(),
var_name='frequency',
type=np.float32,
long_name='Radar frequency',
units='GHz'
)
nc_add_variable(
ds,
val=256,
dimension=(),
var_name='Numfft',
type=np.float32,
long_name='Number of points in FFT',
units='1'
)
if not for_aeris:
nc_add_variable(
ds,
val=np.mean(data['MaxVel']['var']),
dimension=(),
var_name='NyquistVelocity',
type=np.float32,
long_name='Mean (over all chirps) Unambiguous Doppler velocity (+/-)',
units='m s-1'
)
nc_add_variable(
ds,
val=data['Ze']['paraminfo']['altitude'],
dimension=(),
var_name='altitude',
type=np.float32,
long_name='Height of instrument above mean sea level',
units='m'
)
if 'version' in kwargs and cn_version == 'python':
nc_add_variable(
ds,
val=data['no_av'],
dimension=('chirp',),
var_name='NumSpectraAveraged',
type=np.float32,
long_name='Number of spectral averages',
units='1'
)
# time and range variable
# convert to time since start of year
if cn_version == 'python':
ts = np.subtract(data['Ze']['ts'], datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc).timestamp())
ts_str = 'seconds since 2020-01-01 00:00:00 UTC'
ts_unit = f'seconds since 2020-01-01 00:00:00 +00:00 (UTC)'
ts_calendar = 'standard'
rg = data['Ze']['rg']
elif cn_version == 'matlab':
ts = np.subtract(data['Ze']['ts'], datetime.datetime(2001, 1, 1, 0, 0, 0, tzinfo=timezone.utc).timestamp())
ts_str = 'Seconds since 1st January 2001 00:00 UTC'
ts_unit = 'sec'
rg = data['Ze']['rg']
else:
raise ValueError('Wrong version selected! Change version to "matlab" or "python"!')
nc_add_variable(ds, val=ts, dimension=('time',), var_name='time', type=np.float64, long_name=ts_str,
units=ts_unit, calendar=ts_calendar)
nc_add_variable(ds, val=rg, dimension=('range',), var_name='range', type=np.float32,
long_name='Range from antenna to the centre of each range gate', units='m')
if for_aeris:
# time dependent latitude, longitude variables
nc_add_variable(ds, val=data['lat'], dimension=('time',), var_name='latitude', type=np.float32,
long_name='Latitude',
units='degrees_north')
nc_add_variable(ds, val=data['lon'], dimension=('time',), var_name='longitude', type=np.float32,
long_name='Longitude',
units='degrees_east')
else:
# static lat lon variables
nc_add_variable(
ds,
val=data['Ze']['paraminfo']['coordinates'][0],
dimension=(),
var_name='latitude',
type=np.float32,
long_name='latitude',
units='degrees_north'
)
nc_add_variable(
ds,
val=data['Ze']['paraminfo']['coordinates'][1],
dimension=(),
var_name='longitude',
type=np.float32,
long_name='longitude',
units='degrees_east'
)
# chirp dependent variables
if for_aeris:
nc_add_variable(ds, val=data['MaxVel']['var'][0], dimension=('chirp',),
var_name='Nyquist_velocity', type=np.float32, long_name='Unambiguous Doppler velocity (+/-)',
units='m s-1')
else:
nc_add_variable(ds, val=data['MaxVel']['var'][0], dimension=('chirp',),
var_name='DoppMax', type=np.float32,
long_name='Unambiguous Doppler velocity (+/-)',
units='m s-1')
# index plus (1 to n) for Matlab indexing
nc_add_variable(ds, val=data['rg_offsets'], dimension=('chirp',),
var_name='range_offsets', type=np.int32,
long_name='chirp sequences start index array in altitude layer array', units='1')
if for_aeris:
nc_add_variable(ds, val=data['time_shift_array'], dimension=('time', 'chirp'),
var_name='time_shift', type=np.float32,
long_name='Time shift between radar and ship calculated for each hour and chirp', units='s')
# 1D variables
dims_1d = ('time',)
nc_add_variable(ds, val=data['bt']['var'], dimension=dims_1d,
var_name='bt', type=np.float32, long_name='Direct detection brightness temperature at 89 GHz',
units='K')
nc_add_variable(ds, val=data['LWP']['var'], dimension=dims_1d,
var_name='lwp', type=np.float32, long_name='Liquid water path', units='g m-2')
nc_add_variable(ds, val=data['rr']['var'], dimension=dims_1d,
var_name='rain', type=np.float32, long_name='Rain rate from weather station', units='mm h-1')
nc_add_variable(ds, val=data['SurfRelHum']['var'], dimension=dims_1d,
var_name='SurfRelHum', type=np.float32, long_name='Relative humidity from weather station',
units='%')
# 2D variables
nc_add_variable(ds, val=data['Ze']['var'], dimension=dim_tupel, var_name=Ze_str, type=np.float32,
long_name='Radar reflectivity factor', units='dBZ', plot_range=data['Ze']['var_lims'],
plot_scale='linear',
comment='Calibrated reflectivity. Calibration convention: in the absence of attenuation, '
'a cloud at 273 K containing one million 100-micron droplets per cubic metre will '
'have a reflectivity of 0 dBZ at all frequencies.')
nc_add_variable(ds, val=data['VEL_roll']['var'], dimension=dim_tupel, plot_range=data['VEL_roll']['var_lims'],
plot_scale='linear',
var_name=vel_str, type=np.float32, long_name='Best estimate of averaged mean Doppler velocity',
units='m s-1', unit_html='m s<sup>-1</sup>',
comment='This parameter is the radial component of the velocity, with positive velocities are '
'away from the radar (i.e. up) and negative velocities moving towards the radar '
'(i.e. down). It was corrected for the ships heave motion. '
'A rolling average over 3 time steps has been applied to it.',
folding_velocity=data['MaxVel']['var'].max())
if for_aeris:
nc_add_variable(ds, val=data['VEL']['var'], dimension=dim_tupel, plot_range=data['VEL']['var_lims'],
plot_scale='linear',
var_name=f'v_no_rolling_mean_applied', type=np.float32,
long_name='Heave corrected mean Doppler velocity', units='m s-1',
unit_html='m s<sup>-1</sup>',
comment='This parameter is the radial component of the velocity, with positive velocities '
'are away from the radar (i.e. up) and negative velocities moving towards the '
'radar (i.e. down). It was corrected for the ships heave motion but no rolling '
'mean was applied.',
folding_velocity=data['MaxVel']['var'].max())
nc_add_variable(ds, val=data['VEL_uncor']['var'], dimension=dim_tupel,
plot_range=data['VEL_uncor']['var_lims'], plot_scale='linear',
var_name='v_uncor', type=np.float32, long_name='Uncorrected Mean Doppler velocity',
units='m s-1', unit_html='m s<sup>-1</sup>',
comment='This parameter is the uncorrected radial component of the velocity, with positive '
'velocities are away from the radar (i.e. up) and negative velocities moving '
'towards the radar (i.e. down).',
folding_velocity=data['MaxVel']['var'].max())
nc_add_variable(ds, val=data['sw']['var'], dimension=dim_tupel, plot_range=data['sw']['var_lims'],
plot_scale='logarithmic', var_name=width_str, type=np.float32, long_name='Spectral width',
units='m s-1', unit_html='m s<sup>-1</sup>',
comment='This parameter is the standard deviation of the reflectivity-weighted velocities '
'in the radar pulse volume.')
nc_add_variable(ds, val=data['ldr']['var'], dimension=dim_tupel, plot_range=[-30.0, 0.0],
var_name='ldr', type=np.float32, long_name='Linear depolarisation ratio', units='dB',
comment='This parameter is the ratio of cross-polar to co-polar reflectivity.')
nc_add_variable(ds, val=data['kurt']['var'], dimension=dim_tupel, plot_range=data['kurt']['var_lims'],
var_name='kurt', type=np.float32, long_name='Kurtosis', units='1')
nc_add_variable(ds, val=data['skew']['var'], dimension=dim_tupel, plot_range=data['skew']['var_lims'],
var_name='Skew', type=np.float32, long_name='Skewness', units='1')
if for_aeris:
nc_add_variable(ds, val=data['heave_cor'], dimension=dim_tupel, plot_range=data['VEL']['var_lims'],
plot_scale='linear',
var_name='heave_cor', type=np.float32, long_name='Heave rate correction', units='m s-1',
unit_html='m s<sup>-1</sup>',
comment='This is the velocity by which the original Doppler spectrum was corrected by. '
'The heave rate is subtracted from the Doppler velocity, meaning the spectra is '
'shifted to the left for positive heave rates and to the right for negative heave '
'rates.')
nc_add_variable(ds, val=data['heave_cor_bins'], dimension=dim_tupel, plot_scale='linear',
var_name='heave_cor_bins', type=np.int32,
long_name='Heave rate correction in Doppler spectra bins', units='1',
comment='This is the number of bins by which the original Doppler spectrum was shifted by. '
'Positive values shift the spectrum to the left, while negative values shift the '
'spectrum to the right.')
nc_add_variable(ds, val=data['cloud_bases_tops'], dimension=dim_tupel,
var_name='cloud_bases_tops', type=np.int32, long_name='Cloud Bases and Tops', units='1',
comment='Cloud Base: -1, Cloud Top: 1, Else: 0')
nc_add_variable(ds, val=data['hydrometeor_mask'], dimension=dim_tupel,
var_name='hydrometeor_mask', type=np.int32, long_name='Hydrometeor Mask', units='1',
comment='Radar signal: 1, no signal: 0')
print('save calibrated to :: ', ds_name)
return 0
