def export_spectra_to_nc(data, system='', path='', **kwargs): """ This routine generates an hourly NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'. Args: data (dict): dictionary of larda containers system (string): name of the radar system path (string): path where the NetCDF file is stored """ no_chirps = len(data['rg_offsets']) - 1 dt_start = h.ts_to_dt(data['ts'][0]) dt_end = h.ts_to_dt(data['ts'][-1]) ds_name = path + '{}-{}_{}_spectra.nc'.format(dt_start.strftime("%Y%m%d-%H%M"), dt_end.strftime("%H%M"), system) print('open file: ', ds_name) with netCDF4.Dataset(ds_name, "w", format="NETCDF4") as ds: # ds.commit_id = subprocess.check_output(["git", "describe", "--always"]).rstrip() ds.description = '{} calibrated Doppler spectra'.format(system) ds.history = 'Created ' + time.ctime(time.time()) ds.system = system ds.location = data['paraminfo']['location'] ds._FillValue = data['paraminfo']['fill_value'] ds.createDimension('chirp', no_chirps) # add variable number of chirps later ds.createDimension('time', data['ts'].size) for ic in range(no_chirps): ds.createDimension(f'C{ic + 1}range', data['rg'][ic].size) ds.createDimension(f'C{ic + 1}velocity', data['vel'][ic].size) nc_add_variable(ds, val=data['paraminfo']['coordinates'][0], dimension=(), var_name='latitude', type=np.float32, long_name='GPS latitude', unit='deg') nc_add_variable(ds, val=data['paraminfo']['coordinates'][1], dimension=(), var_name='longitude', type=np.float32, long_name='GPS longitude', unit='deg') nc_add_variable(ds, val=data['ts'], dimension=('time',), var_name='time', type=np.float64, long_name='Unix Time - seconds since 01.01.1970 00:00 UTC', unit='sec') nc_add_variable(ds, val=data['rg_offsets'][:no_chirps], dimension=('chirp',), var_name='rg_offsets', type=np.float32, long_name='Range Indices when Chirp shifts ', unit='-') for ic in range(no_chirps): nc_add_variable(ds, val=data['rg'][ic], dimension=(f'C{ic + 1}range',), var_name=f'C{ic + 1}range', type=np.float32, long_name='range', unit='m') nc_add_variable(ds, val=data['vel'][ic], dimension=(f'C{ic + 1}velocity',), var_name=f'C{ic + 1}vel', type=np.float32, long_name='velocity', unit='m s-1') nc_add_variable(ds, val=data['var'][ic], dimension=('time', f'C{ic + 1}range', f'C{ic + 1}velocity'), var_name=f'C{ic + 1}Zspec', type=np.float32, long_name=f'Doppler spectrum at vertical+horizontal polarization: Chirp {ic + 1}', unit='mm6 m-3') nc_add_variable(ds, val=data['vel'][ic][-1], dimension=('chirp',), var_name=f'C{ic + 1}DoppMax', type=np.float32, long_name='Unambiguous Doppler velocity (+/-)', unit='m s-1') return 0
def get_time_slicer(ts, f, time_interval): """get time slicer from the time_interval Following options are available 1. time_interval with [ts_begin, ts_end] 2. only one timestamp is selected and the found right one would be beyond the ts range -> argnearest instead searchsorted 3. only one is timestamp """ # select first timestamp right of begin (not left if nearer as above) #print(f'start time {h.ts_to_dt(ts[0])}') it_b = 0 if ts.shape[0] == 1 else np.searchsorted( ts, h.dt_to_ts(time_interval[0]), side='right') if len(time_interval) == 2: it_e = h.argnearest(ts, h.dt_to_ts(time_interval[1])) if it_b == ts.shape[0]: it_b = it_b - 1 valid_step = 3 * np.median(np.diff(ts)) if ts[it_e] < h.dt_to_ts( time_interval[0]) - valid_step or ts[it_b] < h.dt_to_ts( time_interval[0]): # second condition is to ensure that no timestamp before # the selected interval is choosen # (problem with limrad after change of sampling frequency) str = 'found last profile of file {}\n at ts[it_e] {} too far ({}s) from {}\n'.format( f, h.ts_to_dt(ts[it_e]), valid_step, time_interval[0]) \ + 'or begin too early {} < {}\n returning None'.format(h.ts_to_dt(ts[it_b]), time_interval[0]) logger.warning(str) return None it_e = it_e + 1 if not it_e == ts.shape[0] - 1 else None slicer = [slice(it_b, it_e)] elif it_b == ts.shape[0]: # only one timestamp is selected # and the found right one would be beyond the ts range it_b = h.argnearest(ts, h.dt_to_ts(time_interval[0])) slicer = [slice(it_b, it_b + 1)] else: slicer = [slice(it_b, it_b + 1)] return slicer
def to_text(data_cont): """represent the tree as a multiline string""" maxind_at_level = np.cumsum([2**n for n in range(5)]) - 1 single_tree = data_cont['var'] no_levels = np.searchsorted(maxind_at_level, max(single_tree.keys())) print("no levels", no_levels) dt = h.ts_to_dt(data_cont['ts']) lines = [] lines.append("tree at ts {} and rg {:.2f}".format( dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3], data_cont['rg'])) header = "id (bounds) " + (no_levels)*" " + \ " Z v σ 𝛾" if single_tree[0]['ldr'] != -99: header += " ldr ldrmax" header += " thres prom" lines.append(header) # iterate over children depth first, to build a proper tree for i in child_iter(list(single_tree.keys()))(0): moms = single_tree[i] #print(i, moms["bounds"]) level = np.searchsorted(maxind_at_level, i) spc_bef = "" if level > 0: if np.floor((i - 1) / 2.) % 2 or np.floor((i - 1) / 2.) == 0: if i % 2: spc_bef = " " + (level - 1) * "| " + "+-" else: spc_bef = " " + (level - 1) * "| " + "`-" else: if i % 2: spc_bef = " " + (level - 2) * "| " + " +-" else: spc_bef = " " + (level - 2) * "| " + " `-" spc_aft = (no_levels - level) * " " bounds_f = '({:>3d}, {:>3d})'.format(*moms['bounds']) momstr = '{:> 6.2f}, {:> 6.2f}, {:>5.2f}, {:> 3.2f}'.format( moms['z'], moms['v'], moms['width'], moms['skew']) if moms['ldr'] != -99: momstr += ", {:> 5.1f}, {:> 5.1f}".format(moms['ldr'], moms['ldrmax']) momstr += ", {:> 5.1f}, {:> 5.1f}".format(moms['thres'], moms['prominence']) txt = "{}{:>2d} {}{} | {}".format(spc_bef, i, bounds_f, spc_aft, momstr) lines.append(txt) return '\n'.join(lines)
def __init__(self, _time, _range, *_vel): # build xarray dataset super().__init__() # set metadata if _time is not None: self.attrs['ts_unit'], self.attrs['ts_unit_long'] = 'sec', 'Unix time, seconds since Jan 1. 1979' self.attrs['dt_unit'], self.attrs['dt_unit_long'] = 'date', 'Datetime format' self.coords['ts'] = _time self.coords['dt'] = [h.ts_to_dt(ts) for ts in _time] if _range is not None: self.attrs['rg_unit'], self.attrs['rg_unit_long'] = 'm', 'Meter' self.coords['rg'] = _range # use cloudnet time and range resolution as default if len(_vel) > 0: self.coords['vel'] = _vel[0] if len(_vel) > 1: self.coords.update({f'vel_{ic + 1}': _vel[ic] for ic in range(1, len(_vel))})
def plot_time_spectrogram(ZSpec, rg, **font_settings): rg0 = np.float64(rg) Z = ZSpec.sel(rg=slice(rg0, rg0 + 40.0), ts=slice(ZSpec.ts[0], ZSpec.ts[-1])) z_var = Z.values.copy() #z_var = np.ma.masked_less_equal(Z.values, 0.0) z_var = np.squeeze(z_var) print(z_var.shape) y_var = np.linspace(0, 6 * 256, num=6 * 256) z_var = z_var.reshape((-1, 6 * 256), order='F') fig, ax = plt.subplots(1, figsize=_FIG_SIZE) surf = ax.pcolormesh([h.ts_to_dt(ts) for ts in Z.ts], y_var, z_var[:, :].T, vmin=dBZ_lim[0], vmax=dBZ_lim[1], cmap='jet') ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter('%H:%M:%S')) ax.set_xlabel(xlabel=r"time [UTC]") ax.grid() # Add a color bar which maps values to colors. cbar = fig.colorbar( surf, fraction=0.2, shrink=1., pad=0.01, orientation='vertical', ) cbar.ax.tick_params(axis='both', which='major', labelsize=12, width=2, length=4) cbar.ax.tick_params(axis='both', which='minor', width=2, length=3) cbar.ax.minorticks_on() cbar.ax.set_ylabel("signal normalized", **{}) plt.tight_layout()
def t_r(f, time_interval, *further_intervals): """function that converts the trace netCDF to the data container """ logger.debug("filename at reader {}".format(f)) with netCDF4.Dataset(f, 'r') as ncD: times = ncD.variables[paraminfo['time_variable']][:].astype( np.float64) timeconverter, _ = h.get_converter_array( paraminfo['time_conversion'], ncD=ncD) ts = timeconverter(times) #print('timestamps ', ts[:5]) # setup slice to load base on time_interval it_b = h.argnearest(ts, h.dt_to_ts(time_interval[0])) if len(time_interval) == 2: it_e = h.argnearest(ts, h.dt_to_ts(time_interval[1])) if ts[it_e] < h.dt_to_ts( time_interval[0]) - 3 * np.median(np.diff(ts)): logger.warning( 'last profile of file {}\n at {} too far from {}'. format(f, h.ts_to_dt(ts[it_e]), time_interval[0])) return None it_e = it_e + 1 if not it_e == ts.shape[0] - 1 else None slicer = [slice(it_b, it_e)] else: slicer = [slice(it_b, it_b + 1)] print(slicer) range_interval = further_intervals[0] ranges = ncD.variables[paraminfo['range_variable']] logger.debug('loader range conversion {}'.format( paraminfo['range_conversion'])) rangeconverter, _ = h.get_converter_array( paraminfo['range_conversion'], altitude=paraminfo['altitude']) ir_b = h.argnearest(rangeconverter(ranges[:]), range_interval[0]) if len(range_interval) == 2: if not range_interval[1] == 'max': ir_e = h.argnearest(rangeconverter(ranges[:]), range_interval[1]) ir_e = ir_e + 1 if not ir_e == ranges.shape[0] - 1 else None else: ir_e = None slicer.append(slice(ir_b, ir_e)) else: slicer.append(slice(ir_b, ir_b + 1)) varconverter, maskconverter = h.get_converter_array( paraminfo['var_conversion']) its = np.arange(ts.shape[0])[tuple(slicer)[0]] irs = np.arange(ranges.shape[0])[tuple(slicer)[1]] var = np.empty((its.shape[0], irs.shape[0])) mask = np.empty((its.shape[0], irs.shape[0])) mask[:] = False var = ncD.variables[paraminfo['variable_name']][ tuple(slicer)[0], tuple(slicer)[1], :] data = {} data['dimlabel'] = ['time', 'range', 'cat'] data["filename"] = f data["paraminfo"] = paraminfo data['ts'] = ts[tuple(slicer)[0]] data['system'] = paraminfo['system'] data['name'] = paraminfo['paramkey'] data['colormap'] = paraminfo['colormap'] if 'meta' in paraminfo: data['meta'] = NcReader.get_meta_from_nc( ncD, paraminfo['meta'], paraminfo['variable_name']) variable = ncD.variables[paraminfo['variable_name']] var_definition = ast.literal_eval( variable.getncattr(paraminfo['identifier_var_def'])) if var_definition[1] == "forrest": var_definition[1] = "forest" data['var_definition'] = var_definition data['rg'] = rangeconverter(ranges[tuple(slicer)[1]]) data['rg_unit'] = NcReader.get_var_attr_from_nc( "identifier_rg_unit", paraminfo, ranges) logger.debug('shapes {} {} {}'.format(ts.shape, ranges.shape, var.shape)) data['var_unit'] = NcReader.get_var_attr_from_nc( "identifier_var_unit", paraminfo, var) data['var_lims'] = [float(e) for e in \ NcReader.get_var_attr_from_nc("identifier_var_lims", paraminfo, var)] data['var'] = varconverter(var) data['mask'] = maskconverter(mask) return data
def plot_gn_2d(airmass_source, plottop=12000, xright=7000, xlbl_time='%H:%M', norm=None): #f, parameter, dt_list, dsp, config, savepath, config_dict, model): time_list = airmass_source['ts'] dt_list = [h.ts_to_dt(time) for time in time_list] height_list = airmass_source['rg'] / 1000. no_plots = len(dt_list) if no_plots > 9: xsize = 15 elif no_plots < 8: xsize = no_plots * 1.8 else: xsize = 12 if norm: airmass_source['var'][:, :, :] = airmass_source['var'][:, :, :].copy( ) / norm dsp_text = 'Norm. residence time' else: dsp_text = 'acc. residence time' # fig, axes = plt.subplots(1, no_plots, sharex=True, sharey=True, # figsize=(xsize, 6)) axes = [] fig = plt.figure(constrained_layout=False, figsize=(xsize, 6)) gs1 = fig.add_gridspec(nrows=9, ncols=no_plots, wspace=0.0) for i in range(no_plots): axes.append(fig.add_subplot(gs1[1:, i])) colors = [ (0.65098039215686276, 0.84705882352941175, 0.32941176470588235, 1.0), (1.0, 0.85098039215686272, 0.18431372549019609, 1.0), (0.89803921568627454, 0.7686274509803922, 0.58039215686274515, 1.0), (0.40000000000000002, 0.76078431372549016, 0.6470588235294118, 1.0), (0.9882352941176471, 0.55294117647058827, 0.3843137254901961, 1.0), (0.55294117647058827, 0.62745098039215685, 0.79607843137254897, 1.0), (0.70196078431372544, 0.70196078431372544, 0.70196078431372544, 1.0) ] for it, dt in enumerate(dt_list): occ_height = airmass_source['var'][it, :, :] occ_left = np.cumsum(occ_height, axis=1) geo_names = airmass_source['var_definition'] l = [] for i in range(len(geo_names)): if i == 0: l.append(axes[it].barh(height_list, occ_height[:, 0].T, left=0, align='center', height=0.3, color=colors[0], edgecolor='none')) else: l.append(axes[it].barh(height_list, occ_height[:, i].T, left=occ_left[:, i - 1].T, align='center', height=0.3, color=colors[i], edgecolor='none')) #axes[it].set_ylim([0, 12]) axes[it].set_ylim([0, plottop / 1000.]) axes[it].tick_params(axis='y', which='major', labelsize=14, width=1.5, length=3) axes[it].tick_params(axis='both', which='minor', width=1, length=2) axes[it].tick_params(axis='both', which='both', right=True, top=True, direction='in') axes[it].yaxis.set_minor_locator( matplotlib.ticker.MultipleLocator(1.0)) axes[it].set_xlabel(dt.strftime(xlbl_time), fontsize=14) axes[it].set_xlim([0, xright]) if xright < 20000: axes[it].xaxis.set_major_locator( matplotlib.ticker.MultipleLocator(3000)) else: # axes[it].xaxis.set_major_locator(matplotlib.ticker.FixedLocator([0, xright/2.])) axes[it].xaxis.set_major_locator( matplotlib.ticker.MaxNLocator(nbins=1)) if norm: axes[it].xaxis.set_major_locator(matplotlib.ticker.AutoLocator()) axes[it].xaxis.set_minor_locator(matplotlib.ticker.AutoMinorLocator()) axes[it].tick_params(axis='x', labeltop=False, labelbottom=False) axes[it].tick_params(axis='y', labelleft=False) axes[0].set_ylabel("Height [km]", fontweight='semibold', fontsize=15) axes[0].tick_params(axis='y', labelleft=True) axes[-1].tick_params(axis='x', labeltop=True, labelbottom=False, labelsize=13) plt.suptitle("{} {} {}".format(dt.strftime("%Y%m%d"), airmass_source['paraminfo']['location'], airmass_source['name']), fontweight='semibold', fontsize=15) axes[-1].annotate("Endpoint: {:.1f} {:.1f} ".format( airmass_source['paraminfo']['coordinates'][1], airmass_source['paraminfo']['coordinates'][0]), xy=(.88, 0.92), xycoords='figure fraction', horizontalalignment='center', verticalalignment='bottom', fontsize=12) axes[0].annotate('Time UTC', xy=(.5, .02), xycoords='figure fraction', horizontalalignment='center', verticalalignment='bottom', fontsize=15, fontweight='semibold') axes[-1].annotate(dsp_text, xy=(.87, 0.845), xycoords='figure fraction', horizontalalignment='center', verticalalignment='bottom', fontsize=13, fontweight='semibold') fig.legend( l, list(geo_names.values()), loc='upper left', #bbox_to_anchor=(0.01, 0.952), bbox_to_anchor=(0.07, 0.94), columnspacing=1, ncol=4, fontsize=14, framealpha=0.8) return fig, axes
def plot_ls_2d(airmass_source, plottop=12000, xright=7000, xlbl_time='%H:%M', norm=None): #f, parameter, dt_list, dsp, config, savepath, config_dict, model): time_list = airmass_source['ts'] dt_list = [h.ts_to_dt(time) for time in time_list] height_list = airmass_source['rg'] / 1000. no_plots = len(dt_list) if no_plots > 9: xsize = 15 elif no_plots < 8: xsize = no_plots * 1.8 else: xsize = 12 if norm: airmass_source['var'][:, :, :] = airmass_source['var'][:, :, :].copy( ) / norm dsp_text = 'Norm. residence time' else: dsp_text = 'acc. residence time' axes = [] fig = plt.figure(constrained_layout=False, figsize=(xsize, 6)) gs1 = fig.add_gridspec(nrows=9, ncols=no_plots, wspace=0.0) for i in range(no_plots): axes.append(fig.add_subplot(gs1[1:, i])) #fig, axes = plt.subplots(1, no_plots, sharex=True, sharey=True, figsize=(9, 6)) ls_colors = [ 'lightskyblue', 'darkgreen', 'khaki', 'palegreen', 'red', 'gray', 'tan' ] ls_colors = [ 'lightskyblue', 'seagreen', 'khaki', '#6edd6e', 'darkmagenta', 'gray', 'tan' ] for it, dt in enumerate(dt_list): occ_height = airmass_source['var'][it, :, :] occ_left = np.cumsum(occ_height, axis=1) categories = airmass_source['var_definition'] l1 = axes[it].barh(height_list, occ_height[:, 0].T, align='center', height=0.35, color=ls_colors[0], edgecolor='none') l2 = axes[it].barh(height_list, occ_height[:, 1].T, left=occ_left[:, 0].T, align='center', height=0.35, color=ls_colors[1], edgecolor='none') l3 = axes[it].barh(height_list, occ_height[:, 2].T, left=occ_left[:, 1].T, align='center', height=0.35, color=ls_colors[2], edgecolor='none') l4 = axes[it].barh(height_list, occ_height[:, 3].T, left=occ_left[:, 2].T, align='center', height=0.35, color=ls_colors[3], edgecolor='none') l5 = axes[it].barh(height_list, occ_height[:, 4].T, left=occ_left[:, 3].T, align='center', height=0.35, color=ls_colors[4], edgecolor='none') l6 = axes[it].barh(height_list, occ_height[:, 5].T, left=occ_left[:, 4].T, align='center', height=0.35, color=ls_colors[5], edgecolor='none') l7 = axes[it].barh(height_list, occ_height[:, 6].T, left=occ_left[:, 5].T, align='center', height=0.35, color=ls_colors[6], edgecolor='none') #axes[it].set_ylim([0, 12]) axes[it].set_ylim([0, plottop / 1000.]) axes[it].tick_params(axis='y', which='major', labelsize=14, width=1.5, length=3) axes[it].tick_params(axis='both', which='minor', width=1, length=2) axes[it].tick_params(axis='both', which='both', right=True, top=True, direction='in') axes[it].yaxis.set_minor_locator( matplotlib.ticker.MultipleLocator(1.0)) axes[it].set_xlabel(dt.strftime(xlbl_time), fontsize=14) axes[it].set_xlim(right=xright) if xright < 20000: axes[it].xaxis.set_major_locator( matplotlib.ticker.MultipleLocator(3000)) else: axes[it].xaxis.set_major_locator( matplotlib.ticker.MaxNLocator(nbins=1)) if norm: axes[it].xaxis.set_major_locator(matplotlib.ticker.AutoLocator()) axes[it].xaxis.set_minor_locator(matplotlib.ticker.AutoMinorLocator()) axes[it].tick_params(axis='x', labeltop=False, labelbottom=False) axes[it].tick_params(axis='y', labelleft=False) axes[0].set_ylabel("Height [km]", fontweight='semibold', fontsize=15) axes[0].tick_params(axis='y', labelleft=True) axes[-1].tick_params(axis='x', labeltop=True, labelbottom=False, labelsize=13) plt.suptitle("{} {} {}".format(dt.strftime("%Y%m%d"), airmass_source['paraminfo']['location'], airmass_source['name']), fontweight='semibold', fontsize=15) axes[-1].annotate("Endpoint: {:.1f} {:.1f} ".format( airmass_source['paraminfo']['coordinates'][1], airmass_source['paraminfo']['coordinates'][0]), xy=(.88, 0.92), xycoords='figure fraction', horizontalalignment='center', verticalalignment='bottom', fontsize=12) axes[0].annotate('Time UTC', xy=(.5, .02), xycoords='figure fraction', horizontalalignment='center', verticalalignment='bottom', fontsize=15, fontweight='semibold') axes[-1].annotate(dsp_text, xy=(.87, 0.845), xycoords='figure fraction', horizontalalignment='center', verticalalignment='bottom', fontsize=13, fontweight='semibold') fig.legend( (l1, l2, l3, l4, l5, l6, l7), list(categories.values()), #bbox_to_anchor=(0.85, 0.952), bbox_to_anchor=(0.07, 0.94), loc='upper left', columnspacing=1, ncol=4, fontsize=14, framealpha=0.8) #fig.set_tight_layout({'rect': [0, 0, 1, 1], 'pad': 0.1, 'h_pad': 1.5}) #plt.tight_layout(w_pad=0.0002) return fig, axes
def get_param(campaign_name, system, param): """ """ app.logger.info("got request for {} {} {}".format(campaign_name, system, param)) starttime = time.time() larda = pyLARDA.LARDA().connect(campaign_name, build_lists=False) app.logger.debug("{:5.3f}s load larda".format(time.time() - starttime)) if "rformat" in request.args and request.args['rformat'] == 'bin': rformat = 'bin' elif "rformat" in request.args and request.args['rformat'] == 'msgpack': rformat = 'msgpack' else: rformat = 'json' intervals = request.args.get('interval').split(',') time_interval = [h.ts_to_dt(float(t)) for t in intervals[0].split('-')] further_slices = [[ float(e) if e not in ['max'] else e for e in s.split('-') ] for s in intervals[1:]] app.logger.warning('request.args {}'.format(dict(request.args))) app.logger.info("time request {}".format(time_interval)) starttime = time.time() data_container = larda.read(system, param, time_interval, *further_slices, **dict(request.args)) app.logger.debug("{:5.3f}s read data".format(time.time() - starttime)) starttime = time.time() for k in ['ts', 'rg', 'vel', 'var', 'mask']: if k in data_container and hasattr(data_container[k], 'tolist'): if data_container[k].dtype is not np.dtype('object'): data_container[k][~np.isfinite(data_container[k])] = 0 data_container[k] = data_container[k].tolist() app.logger.debug("{:5.3f}s convert data".format(time.time() - starttime)) #import io #test_datacont = {**data_container} #for k in ['ts', 'rg', 'var', 'mask']: # if k in test_datacont: # print(k,type(test_datacont[k])) # start = time.time() # test_datacont[k][~np.isfinite(test_datacont[k])] = 0 # memfile = io.BytesIO() # np.save(memfile, test_datacont[k]) # memfile.seek(0) # test_datacont[k] = memfile.read().decode('latin-1') #resp = Response(json.dumps(test_datacont), status=200, mimetype='application/json') starttime = time.time() if rformat == 'bin': resp = Response(cbor.dumps(data_container), status=200, mimetype='application/cbor') elif rformat == 'msgpack': resp = Response(msgpack.packb(data_container), status=200, mimetype='application/msgpack') elif rformat == 'json': resp = Response(json.dumps(data_container), status=200, mimetype='application/json') app.logger.debug("{:5.3f}s dumps {}".format(time.time() - starttime, rformat)) #for some reason the manual response is faster... #return jsonify(data_container) return resp
def pt_ret(f, time_interval, *further_intervals): """function that converts the peakTree netCDF to the data container """ logger.debug("filename at reader {}".format(f)) with netCDF4.Dataset(f, 'r') as ncD: times = ncD.variables[paraminfo['time_variable']][:].astype(np.float64) if 'time_millisec_variable' in paraminfo.keys() and \ paraminfo['time_millisec_variable'] in ncD.variables: subsec = ncD.variables[paraminfo['time_millisec_variable']][:]/1.0e3 times += subsec if 'time_microsec_variable' in paraminfo.keys() and \ paraminfo['time_microsec_variable'] in ncD.variables: subsec = ncD.variables[paraminfo['time_microsec_variable']][:]/1.0e6 times += subsec timeconverter, _ = h.get_converter_array( paraminfo['time_conversion'], ncD=ncD) ts = timeconverter(times) #print('timestamps ', ts[:5]) # setup slice to load base on time_interval it_b = h.argnearest(ts, h.dt_to_ts(time_interval[0])) if len(time_interval) == 2: it_e = h.argnearest(ts, h.dt_to_ts(time_interval[1])) if ts[it_e] < h.dt_to_ts(time_interval[0])-3*np.median(np.diff(ts)): logger.warning( 'last profile of file {}\n at {} too far from {}'.format( f, h.ts_to_dt(ts[it_e]), time_interval[0])) return None it_e = it_e+1 if not it_e == ts.shape[0]-1 else None slicer = [slice(it_b, it_e)] else: slicer = [slice(it_b, it_b+1)] print(slicer) if paraminfo['ncreader'] == 'peakTree': range_tg = True range_interval = further_intervals[0] ranges = ncD.variables[paraminfo['range_variable']] logger.debug('loader range conversion {}'.format(paraminfo['range_conversion'])) rangeconverter, _ = h.get_converter_array( paraminfo['range_conversion'], altitude=paraminfo['altitude']) ir_b = h.argnearest(rangeconverter(ranges[:]), range_interval[0]) if len(range_interval) == 2: if not range_interval[1] == 'max': ir_e = h.argnearest(rangeconverter(ranges[:]), range_interval[1]) ir_e = ir_e+1 if not ir_e == ranges.shape[0]-1 else None else: ir_e = None slicer.append(slice(ir_b, ir_e)) else: slicer.append(slice(ir_b, ir_b+1)) varconverter, maskconverter = h.get_converter_array( paraminfo['var_conversion']) its = np.arange(ts.shape[0])[tuple(slicer)[0]] irs = np.arange(ranges.shape[0])[tuple(slicer)[1]] var = np.empty((its.shape[0], irs.shape[0]), dtype=object) mask = np.empty((its.shape[0], irs.shape[0]), dtype=bool) mask[:] = True param_list = [ ncD.variables['parent'][tuple(slicer)[0],tuple(slicer)[1],:], #0 ncD.variables['Z'][tuple(slicer)[0],tuple(slicer)[1],:], #1 ncD.variables['v'][tuple(slicer)[0],tuple(slicer)[1],:], #2 ncD.variables['width'][tuple(slicer)[0],tuple(slicer)[1],:], #3 ncD.variables['skew'][tuple(slicer)[0],tuple(slicer)[1],:], #4 ncD.variables['threshold'][tuple(slicer)[0],tuple(slicer)[1],:], #5 ncD.variables['prominence'][tuple(slicer)[0],tuple(slicer)[1],:], #6 ncD.variables['bound_l'][tuple(slicer)[0],tuple(slicer)[1],:], #7 ncD.variables['bound_r'][tuple(slicer)[0],tuple(slicer)[1],:] #8 ] if 'LDR' in ncD.variables.keys(): ldr_avail = True param_list.append(ncD.variables['LDR'][tuple(slicer)[0],tuple(slicer)[1],:]) #9 param_list.append(ncD.variables['ldrmax'][tuple(slicer)[0],tuple(slicer)[1],:]) #10 else: ldr_avail = False data = np.stack(tuple(param_list), axis=3) print(data.shape) if fastbuilder: var, mask = peakTree_fastbuilder.array_to_tree_c(data.astype(float), ldr_avail) else: var, mask = array_to_tree_py(data, ldr_avail) data = {} data['dimlabel'] = ['time', 'range', 'dict'] data["filename"] = f data["paraminfo"] = paraminfo data['ts'] = ts[tuple(slicer)[0]] data['system'] = paraminfo['system'] data['name'] = paraminfo['paramkey'] data['colormap'] = paraminfo['colormap'] data['rg'] = rangeconverter(ranges[tuple(slicer)[1]]) data['rg_unit'] = NcReader.get_var_attr_from_nc("identifier_rg_unit", paraminfo, ranges) logger.debug('shapes {} {} {}'.format(ts.shape, ranges.shape, var.shape)) logger.debug('shapes {} {}'.format(ts.shape, var.shape)) data['var_unit'] = NcReader.get_var_attr_from_nc("identifier_var_unit", paraminfo, var) data['var_lims'] = [float(e) for e in \ NcReader.get_var_attr_from_nc("identifier_var_lims", paraminfo, var)] data['var'] = varconverter(var) data['mask'] = maskconverter(mask) return data
def export_spectra2nc(data, larda_git_path='', system='', path='', **kwargs): """ This routine generates an hourly NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'. Args: data (dict): dictionary of larda containers system (string): name of the radar system path (string): path where the NetCDF file is stored """ hour_bias = kwargs['hour_bias'] if 'hour_bias' in kwargs else 0 dt_start = h.ts_to_dt(data['time'][0]) dt_end = h.ts_to_dt(data['time'][-1]) ds_name = path + f'{dt_start:%Y%m%d-%H%M}-{dt_end:%H%M}_{system}_spectra.nc' repo = git.Repo(larda_git_path) sha = repo.head.object.hexsha print('open file: ', ds_name) with netCDF4.Dataset(ds_name, "w", format="NETCDF4") as ds: ds.git_description = f'GIT commit ID {sha}' ds.description = f'{system} calibrated Doppler spectra' ds.history = 'Created ' + time.ctime(time.time()) ds.system = system ds.location = data['location'] ds._FillValue = data['fill_value'] ds.createDimension( 'chirp', data['chirps']) # add variable number of chirps later ds.createDimension('time', data['time'].size) ds.createDimension('range', data['range'].size) ds.createDimension('velocity', data['velocity'].shape[1]) nc_add_variable(ds, val=data['coordinates'][0], dimension=(), var_name='latitude', type=np.float32, long_name='GPS latitude', unit='deg') nc_add_variable(ds, val=data['coordinates'][1], dimension=(), var_name='longitude', type=np.float32, long_name='GPS longitude', unit='deg') nc_add_variable(ds, val=data['altitude'], dimension=(), var_name='altitude', type=np.float32, long_name='altitude, i.e. height above sea level', unit='m') nc_add_variable( ds, val=data['time'], dimension=('time', ), var_name='time', type=np.float64, long_name='Unix Time - seconds since 01.01.1970 00:00 UTC', unit='sec') nc_add_variable(ds, val=data['rg_offsets'], dimension=('chirp', ), var_name='rg_offsets', type=np.float32, long_name='Range Indices for next chirp sequence.', unit='-') nc_add_variable(ds, val=data['range'], dimension=('range', ), var_name='range', type=np.float32, long_name='range', unit='m') nc_add_variable(ds, val=data['velocity'], dimension=( 'chirp', 'velocity', ), var_name=f'velocity_vectors', type=np.float32, long_name='velocity vectors for each chirp', unit='m s-1') nc_add_variable(ds, val=data['nyquist_velocity'], dimension=('chirp', ), var_name='nyquist_velocity', type=np.float32, long_name='Unambiguous Doppler velocity (+/-)', unit='m s-1') nc_add_variable( ds, val=data['doppler_spectrum'], dimension=('time', 'range', 'velocity'), var_name='doppler_spectrum', type=np.float32, long_name= 'Doppler spectrum, if dual polarization radar: doppler_spectrum = vertical + horizontal polarization', unit='mm6 m-3 (m s-1)-1') try: nc_add_variable(ds, val=data['covariance_spectrum_re'], dimension=('time', 'range', 'velocity'), var_name='covariance_spectrum_re', type=np.float32, long_name='Real part of covariance spectrum', unit='mm6 m-3') except KeyError: print('skip writing real part of covariance spectrum') try: nc_add_variable( ds, val=data['covariance_spectrum_im'], dimension=('time', 'range', 'velocity'), var_name='covariance_spectrum_im', type=np.float32, long_name='Imaginary part of covariance spectrum', unit='mm6 m-3') except KeyError: print('skip writing imaginary part of covariance spectrum') try: nc_add_variable( ds, val=data['sensitivity_limit'], dimension=('time', 'range'), var_name='sensitivity_limit', type=np.float32, long_name= 'Sensitivity limit, if dual polarization radar: sensitivity_limit = vertical + horizontal polarization', unit='mm6 m-3') except KeyError: print('skip writing sensitivity limit') try: nc_add_variable( ds, val=data['doppler_spectrum_h'], dimension=('time', 'range', 'velocity'), var_name='doppler_spectrum_h', type=np.float32, long_name='Doppler spectrum, horizontal polarization only', unit='mm6 m-3 ') nc_add_variable( ds, val=data['sensitivity_limit_h'], dimension=('time', 'range'), var_name='sensitivity_limit_h', type=np.float32, long_name='Sensitivity limit for horizontal polarization', unit='mm6 m-3 (m s-1)-1') except KeyError: print('Skip writing horizontal polarization.') return 0
def rpg_radar2nc_old(data, path, **kwargs): """ This routine generates a daily NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'. Args: data (dict): dictionary of larda containers path (string): path where the NetCDF file is stored """ import time no_chirps = 3 if 'time_frame' in kwargs: ds_name = path + kwargs['time_frame'] + '_LIMRAD94.nc'.format( h.ts_to_dt(data['Ze']['ts'][0]).strftime("%Y%m%d"), kwargs['time_frame']) else: ds_name = path + '{}_000000-240000_LIMRAD94.nc'.format( h.ts_to_dt(data['Ze']['ts'][0]).strftime("%Y%m%d")) with netCDF4.Dataset(ds_name, "w", format="NETCDF4") as ds: # ds.commit_id = subprocess.check_output(["git", "describe", "--always"]) .rstrip() ds.description = 'Concatenated data files of LIMRAD 94GHz - FMCW Radar, used as input for Cloudnet processing, ' \ 'filters applied: ghos-echo, despeckle, use only main peak' ds.history = 'Created ' + time.ctime(time.time()) ds.source = data['Ze']['paraminfo']['location'] ds._FillValue = data['Ze']['paraminfo']['fill_value'] ds.createDimension('chirp', no_chirps) # add variable number of chirps later ds.createDimension('time', data['Ze']['ts'].size) ds.createDimension('range', data['Ze']['rg'].size) # coordinates nc_add_variable(ds, val=data['Ze']['paraminfo']['coordinates'][0], dimension=(), var_name='latitude', type=np.float32, long_name='GPS latitude', unit='deg') nc_add_variable(ds, val=data['Ze']['paraminfo']['coordinates'][1], dimension=(), var_name='longitude', type=np.float32, long_name='GPS longitude', unit='deg') # time and range variable # convert to time since 20010101 ts = np.subtract(data['Ze']['ts'], datetime.datetime(2001, 1, 1, 0, 0, 0).timestamp()) nc_add_variable(ds, val=ts, dimension=('time', ), var_name='time', type=np.float64, long_name='Seconds since 01.01.2001 00:00 UTC', unit='sec') nc_add_variable(ds, val=data['Ze']['rg'], dimension=('range', ), var_name='range', type=np.float32, long_name='range', unit='m') # 2D variables nc_add_variable(ds, val=data['Ze']['var'].T, dimension=( 'range', 'time', ), var_name='Ze', type=np.float32, long_name='Equivalent radar reflectivity factor', unit='mm^6/m^3') nc_add_variable(ds, val=data['VEL']['var'].T, dimension=( 'range', 'time', ), var_name='vm', type=np.float32, long_name='Mean Doppler velocity', unit='m/s') nc_add_variable(ds, val=data['sw']['var'].T, dimension=( 'range', 'time', ), var_name='sigma', type=np.float32, long_name='Spectrum width', unit='m/s') nc_add_variable(ds, val=data['ldr']['var'].T, dimension=( 'range', 'time', ), var_name='ldr', type=np.float32, long_name='Slanted linear depolarization ratio', unit='dB') nc_add_variable(ds, val=data['kurt']['var'].T, dimension=( 'range', 'time', ), var_name='kurt', type=np.float32, long_name='Kurtosis', unit='[linear]') nc_add_variable(ds, val=data['skew']['var'].T, dimension=( 'range', 'time', ), var_name='Skew', type=np.float32, long_name='Skewness', unit='[linear]') nc_add_variable(ds, val=data['DiffAtt']['var'].T, dimension=( 'range', 'time', ), var_name='DiffAtt', type=np.float32, long_name='Differential attenuation', unit='dB/km') # 1D variables nc_add_variable(ds, val=data['bt']['var'], dimension=('time', ), var_name='bt', type=np.float32, long_name='Direct detection brightness temperature', unit='K') nc_add_variable(ds, val=data['LWP']['var'], dimension=('time', ), var_name='lwp', type=np.float32, long_name='Liquid water path', unit='g/m^2') nc_add_variable(ds, val=data['rr']['var'], dimension=('time', ), var_name='rain', type=np.float32, long_name='Rain rate from weather station', unit='mm/h') nc_add_variable(ds, val=data['SurfRelHum']['var'], dimension=('time', ), var_name='SurfRelHum', type=np.float32, long_name='Relative humidity from weather station', unit='%') # chirp dependent variables nc_add_variable(ds, val=data['MaxVel']['var'][0], dimension=('chirp', ), var_name='DoppMax', type=np.float32, long_name='Unambiguous Doppler velocity (+/-)', unit='m/s') range_offsets = np.ones(no_chirps, dtype=np.uint) for iC in range(no_chirps - 1): range_offsets[iC + 1] = range_offsets[iC] + data[ 'C' + str(iC + 1) + 'Range']['var'][0].shape nc_add_variable( ds, val=range_offsets, dimension=('chirp', ), var_name='range_offsets', type=np.uint, long_name= 'chirp sequences start index array in altitude layer array', unit='[-]') print('save calibrated to :: ', ds_name) return 0
def rpg_radar2nc(data, path, larda_git_path, **kwargs): """ This routine generates a daily NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'. Args: data (dict): dictionary of larda containers path (string): path where the NetCDF file is stored """ dt_start = h.ts_to_dt(data['Ze']['ts'][0]) h.make_dir(path) site_name = kwargs['site'] if 'site' in kwargs else 'no-site' hour_bias = kwargs['hour_bias'] if 'hour_bias' in kwargs else 0 cn_version = kwargs['version'] if 'version' in kwargs else 'pyhon' ds_name = f'{path}/{h.ts_to_dt(data["Ze"]["ts"][0]):%Y%m%d}-{site_name}-limrad94.nc' ncvers = '4' repo = git.Repo(larda_git_path) sha = repo.head.object.hexsha with netCDF4.Dataset(ds_name, 'w', format=f'NETCDF{ncvers}') as ds: ds.Convention = 'CF-1.0' ds.location = data['Ze']['paraminfo']['location'] ds.system = data['Ze']['paraminfo']['system'] ds.version = f'Variable names and dimensions prepared for Cloudnet {kwargs["version"]} version' ds.title = 'LIMRAD94 (SLDR) Doppler Cloud Radar, calibrated Input for Cloudnet' ds.institution = 'Leipzig Institute for Meteorology (LIM), Leipzig, Germany' ds.source = '94 GHz Cloud Radar LIMRAD94\nRadar type: Frequency Modulated Continuous Wave,\nTransmitter power 1.5 W typical (solid state ' \ 'amplifier)\nAntenna Type: Bi-static Cassegrain with 500 mm aperture\nBeam width: 0.48deg FWHM' ds.reference = 'W Band Cloud Radar LIMRAD94\nDocumentation and User Manual provided by manufacturer RPG Radiometer Physics GmbH\n' \ 'Information about system also available at https://www.radiometer-physics.de/' ds.calibrations = f'remove Precip. ghost: {kwargs["ghost_echo_1"]}\n, remove curtain ghost: {kwargs["ghost_echo_2"]}\n' \ f'despeckle: {kwargs["despeckle"]}\n, number of standard deviations above noise: {kwargs["NF"]}\n' ds.git_description = f'GIT commit ID {sha}' ds.description = 'Concatenated data files of LIMRAD 94GHz - FMCW Radar, used as input for Cloudnet processing, ' \ 'filters applied: ghos-echo, despeckle, use only main peak' ds.history = 'Created ' + time.ctime(time.time()) ds._FillValue = data['Ze']['paraminfo']['fill_value'] ds.day = dt_start.day ds.month = dt_start.month ds.year = dt_start.year # ds.commit_id = subprocess.check_output(["git", "describe", "--always"]) .rstrip() ds.history = 'Created ' + time.ctime(time.time( )) + '\nfilters applied: ghost-echo, despeckle, main peak only' Ze_str = 'Zh' if cn_version == 'python' else 'Ze' vel_str = 'v' if cn_version == 'python' else 'vm' width_str = 'width' if cn_version == 'python' else 'sigma' dim_tupel = ('time', 'range') if cn_version == 'python' else ('range', 'time') n_chirps = len(data['no_av']) ds.createDimension('chirp', n_chirps) ds.createDimension('time', data['Ze']['ts'].size) ds.createDimension('range', data['Ze']['rg'].size) if cn_version == 'matlab': for ivar in ['Ze', 'VEL', 'sw', 'ldr', 'kurt', 'skew']: data[ivar]['var'] = data[ivar]['var'].T # coordinates nc_add_variable(ds, val=94.0, dimension=(), var_name='frequency', type=np.float32, long_name='Radar frequency', units='GHz') nc_add_variable(ds, val=256, dimension=(), var_name='Numfft', type=np.float32, long_name='Number of points in FFT', units='') nc_add_variable( ds, val=np.mean(data['MaxVel']['var']), dimension=(), var_name='NyquistVelocity', type=np.float32, long_name= 'Mean (over all chirps) Unambiguous Doppler velocity (+/-)', units='m s-1') nc_add_variable(ds, val=data['Ze']['paraminfo']['altitude'], dimension=(), var_name='altitude', type=np.float32, long_name='Height of instrument above mean sea level', units='m') nc_add_variable(ds, val=data['Ze']['paraminfo']['coordinates'][0], dimension=(), var_name='latitude', type=np.float32, long_name='latitude', units='degrees_north') nc_add_variable(ds, val=data['Ze']['paraminfo']['coordinates'][1], dimension=(), var_name='longitude', type=np.float32, long_name='longitude', units='degrees_east') if 'version' in kwargs and cn_version == 'python': nc_add_variable(ds, val=data['no_av'], dimension=('chirp', ), var_name='NumSpectraAveraged', type=np.float32, long_name='Number of spectral averages', units='') # time and range variable # convert to time since midnight if cn_version == 'python': ts = np.subtract( data['Ze']['ts'], datetime.datetime(dt_start.year, dt_start.month, dt_start.day, hour_bias, 0, 0).timestamp()) ts_str = 'Decimal hours from midnight UTC to the middle of each day' ts_unit = f'hours since {dt_start:%Y-%m-%d} 00:00:00 +00:00 (UTC)' rg = data['Ze']['rg'] / 1000.0 elif cn_version == 'matlab': ts = np.subtract( data['Ze']['ts'], datetime.datetime(2001, 1, 1, hour_bias, 0, 0).timestamp()) ts_str = 'Seconds since 1st January 2001 00:00 UTC' ts_unit = 'sec' rg = data['Ze']['rg'] else: raise ValueError( 'Wrong version selected! version to "matlab" or "python"!') nc_add_variable(ds, val=ts, dimension=('time', ), var_name='time', type=np.float64, long_name=ts_str, units=ts_unit) nc_add_variable( ds, val=rg, dimension=('range', ), var_name='range', type=np.float32, long_name='Range from antenna to the centre of each range gate', units='km') nc_add_variable(ds, val=data['Azm']['var'], dimension=('time', ), var_name='azimuth', type=np.float32, long_name='Azimuth angle from north', units='degree') nc_add_variable( ds, val=data['Elv']['var'], dimension=('time', ), var_name='elevation', type=np.float32, long_name='elevation angle. 90 degree is vertical direction.', units='degree') # chirp dependent variables nc_add_variable(ds, val=data['MaxVel']['var'][0], dimension=('chirp', ), var_name='DoppMax', type=np.float32, long_name='Unambiguous Doppler velocity (+/-)', units='m s-1') # index plus (1 to n) for Matlab indexing nc_add_variable( ds, val=data['rg_offsets'], dimension=('chirp', ), var_name='range_offsets', type=np.uint32, long_name= 'chirp sequences start index array in altitude layer array', units='-') # 1D variables nc_add_variable(ds, val=data['bt']['var'], dimension=('time', ), var_name='bt', type=np.float32, long_name='Direct detection brightness temperature', units='K') nc_add_variable(ds, val=data['LWP']['var'], dimension=('time', ), var_name='lwp', type=np.float32, long_name='Liquid water path', units='g m-2') nc_add_variable(ds, val=data['rr']['var'], dimension=('time', ), var_name='rain', type=np.float32, long_name='Rain rate from weather station', units='mm h-1') nc_add_variable(ds, val=data['SurfRelHum']['var'], dimension=('time', ), var_name='SurfRelHum', type=np.float32, long_name='Relative humidity from weather station', units='%') # 2D variables nc_add_variable( ds, val=data['Ze']['var'], dimension=dim_tupel, var_name=Ze_str, type=np.float32, long_name='Radar reflectivity factor', units='mm6 m-3', plot_range=data['Ze']['var_lims'], plot_scale='linear', comment= 'Calibrated reflectivity. Calibration convention: in the absence of attenuation, ' 'a cloud at 273 K containing one million 100-micron droplets per cubic metre will ' 'have a reflectivity of 0 dBZ at all frequencies.') nc_add_variable( ds, val=data['VEL']['var'], dimension=dim_tupel, plot_range=data['VEL']['var_lims'], plot_scale='linear', var_name=vel_str, type=np.float32, long_name='Doppler velocity', units='m s-1', unit_html='m s<sup>-1</sup>', comment= 'This parameter is the radial component of the velocity, with positive velocities are away from the radar.', folding_velocity=data['MaxVel']['var'].max()) nc_add_variable( ds, val=data['sw']['var'], dimension=dim_tupel, plot_range=data['sw']['var_lims'], lot_scale='logarithmic', var_name=width_str, type=np.float32, long_name='Spectral width', units='m s-1', unit_html='m s<sup>-1</sup>', comment= 'This parameter is the standard deviation of the reflectivity-weighted velocities in the radar pulse volume.' ) nc_add_variable( ds, val=data['ldr']['var'], dimension=dim_tupel, plot_range=[-30.0, 0.0], var_name='ldr', type=np.float32, long_name='Linear depolarisation ratio', units='dB', comment= 'This parameter is the ratio of cross-polar to co-polar reflectivity.' ) nc_add_variable(ds, val=data['kurt']['var'], dimension=dim_tupel, plot_range=data['kurt']['var_lims'], var_name='kurt', type=np.float32, long_name='Kurtosis', units='linear') nc_add_variable(ds, val=data['skew']['var'], dimension=dim_tupel, plot_range=data['skew']['var_lims'], var_name='Skew', type=np.float32, long_name='Skewness', units='linear') print('save calibrated to :: ', ds_name) return 0
def retfunc(f, time_interval, range_interval): """function that converts the netCDF to the larda-data-format """ logger.debug("filename at reader {}".format(f)) with netCDF4.Dataset(f, 'r') as ncD: ranges = ncD.variables[paraminfo['range_variable']] times = ncD.variables[paraminfo['time_variable']][:].astype( np.float64) locator_mask = ncD.variables[paraminfo['mask_var']][:].astype( np.int) if 'time_millisec_variable' in paraminfo.keys() and \ paraminfo['time_millisec_variable'] in ncD.variables: subsec = ncD.variables[ paraminfo['time_millisec_variable']][:] / 1.0e3 times += subsec if 'time_microsec_variable' in paraminfo.keys() and \ paraminfo['time_microsec_variable'] in ncD.variables: subsec = ncD.variables[ paraminfo['time_microsec_variable']][:] / 1.0e6 times += subsec if 'base_time_variable' in paraminfo.keys() and \ paraminfo['base_time_variable'] in ncD.variables: basetime = ncD.variables[ paraminfo['base_time_variable']][:].astype(np.float64) times += basetime timeconverter, _ = h.get_converter_array( paraminfo['time_conversion'], ncD=ncD) ts = timeconverter(times) it_b = np.searchsorted(ts, h.dt_to_ts(time_interval[0]), side='right') if len(time_interval) == 2: it_e = h.argnearest(ts, h.dt_to_ts(time_interval[1])) if it_b == ts.shape[0]: it_b = it_b - 1 if ts[it_e] < h.dt_to_ts(time_interval[0]) - 3 * np.median(np.diff(ts)) \ or ts[it_b] < h.dt_to_ts(time_interval[0]): # second condition is to ensure that no timestamp before # the selected interval is chosen # (problem with limrad after change of sampling frequency) logger.warning( 'last profile of file {}\n at {} too far from {}'. format(f, h.ts_to_dt(ts[it_e]), time_interval[0])) return None it_e = it_e + 1 if not it_e == ts.shape[0] - 1 else None slicer = [slice(it_b, it_e)] elif it_b == ts.shape[0]: # only one timestamp is selected # and the found right one would be beyond the ts range it_b = h.argnearest(ts, h.dt_to_ts(time_interval[0])) slicer = [slice(it_b, it_b + 1)] else: slicer = [slice(it_b, it_b + 1)] rangeconverter, _ = h.get_converter_array( paraminfo['range_conversion']) varconverter, _ = h.get_converter_array( paraminfo['var_conversion']) ir_b = h.argnearest(rangeconverter(ranges[:]), range_interval[0]) if len(range_interval) == 2: if not range_interval[1] == 'max': ir_e = h.argnearest(rangeconverter(ranges[:]), range_interval[1]) ir_e = ir_e + 1 if not ir_e == ranges.shape[0] - 1 else None else: ir_e = None slicer.append(slice(ir_b, ir_e)) else: slicer.append(slice(ir_b, ir_b + 1)) range_out = rangeconverter(ranges[tuple(slicer)[1]]) cal = getattr(ncD, paraminfo['cal_const']) var = ncD.variables[paraminfo['variable_name']][:].astype( np.float64) var = var[locator_mask] vel = ncD.variables[paraminfo['vel_variable']][:].astype( np.float64) # print('var dict ',ch1var.__dict__) # print('shapes ', ts.shape, ch1range.shape, ch1var.shape) # print("time indices ", it_b, it_e) data = {} data['dimlabel'] = ['time', 'range', 'vel'] data["filename"] = f data["paraminfo"] = paraminfo data['ts'] = ts[tuple(slicer)[0]] data['rg'] = range_out data['system'] = paraminfo['system'] data['name'] = paraminfo['paramkey'] data['colormap'] = paraminfo['colormap'] # also experimental: vis_varconverter if 'plot_varconverter' in paraminfo and paraminfo[ 'plot_varconverter'] != 'none': data['plot_varconverter'] = paraminfo['plot_varconverter'] else: data['plot_varconverter'] = '' data['rg_unit'] = get_var_attr_from_nc("identifier_rg_unit", paraminfo, ranges) #data['var_unit'] = get_var_attr_from_nc("identifier_var_unit", # paraminfo, var) data['var_unit'] = 'dBZ m-1 s' data['var_lims'] = [float(e) for e in \ get_var_attr_from_nc("identifier_var_lims", paraminfo, var)] data['vel'] = vel if "identifier_fill_value" in paraminfo.keys( ) and not "fill_value" in paraminfo.keys(): fill_value = var.getncattr(paraminfo['identifier_fill_value']) data['mask'] = (var[tuple(slicer)].data == fill_value) elif "fill_value" in paraminfo.keys(): fill_value = paraminfo["fill_value"] data['mask'] = np.isclose(var[tuple(slicer)], fill_value) elif "mask_var" in paraminfo.keys(): # combine locator mask and mask of infinite values mask = locator_mask.mask[tuple(slicer)] data["mask"] = np.logical_or( ~np.isfinite(var[tuple(slicer)].data), np.repeat(mask[:, :, np.newaxis], len(data['vel']), axis=2)) else: data['mask'] = ~np.isfinite(var[tuple(slicer)].data) if isinstance(times, np.ma.MaskedArray): var = varconverter(var[tuple(slicer)].data) else: var = varconverter(var[tuple(slicer)]) var2 = h.z2lin(var) * h.z2lin(float( cal[:-3])) * (range_out**2)[np.newaxis, :, np.newaxis] data['var'] = var2 return data
def generate_weather_file_LIMRAD94(data, path, **kwargs): """ This routine generates a daily NetCDF4 file with the weather station measurements from the RPG 94 GHz FMCW radar 'LIMRAD94'. Args: data (dict): dictionary of larda containers path (string): path where the NetCDF file is stored """ import time if 'time_frame' in kwargs: ds_name = path + kwargs['time_frame'] + \ '_LIMRAD94_weather.nc'.format(h.ts_to_dt(data['SurfWS']['ts'][0]).strftime("%Y%m%d"), kwargs['time_frame']) else: ds_name = path + '{}_000000-240000_LIMRAD94_weather.nc'.format(h.ts_to_dt(data['SurfWS']['ts'][0]).strftime("%Y%m%d")) ds = netCDF4.Dataset(ds_name, "w", format="NETCDF4") ds.description = 'Concatenated data files of LIMRAD 94GHz - FMCW Radar, Data from the Vaisalla weather station' ds.history = 'Created ' + time.ctime(time.time()) ds.source = data['SurfWS']['paraminfo']['location'] ds.FillValue = data['SurfWS']['paraminfo']['fill_value'] ds.createDimension('time', data['SurfWS']['ts'].size) # coordinates nc_add_variable(ds, val=data['SurfWS']['paraminfo']['coordinates'][0], dimension=(), var_name='latitude', type=np.float32, long_name='GPS latitude', unit='deg') nc_add_variable(ds, val=data['SurfWS']['paraminfo']['coordinates'][1], dimension=(), var_name='longitude', type=np.float32, long_name='GPS longitude', unit='deg') # time # convert to time since 20010101 ts = np.subtract(data['SurfWS']['ts'], datetime.datetime(2001, 1, 1, 0, 0, 0).timestamp()) nc_add_variable(ds, val=ts, dimension=('time',), var_name='time', type=np.float64, long_name='Seconds since 2001-01-01 00:00 UTC', unit='seconds') # 1D variables nc_add_variable(ds, val=data['SurfRelHum']['var'], dimension=('time',), var_name='SurfRelHum', type=np.float32, long_name='Relative humidity from weather station', unit='%') nc_add_variable(ds, val=data['SurfTemp']['var'], dimension=('time',), var_name='SurfTemp', type=np.float32, long_name='Surface temperature from weather station', unit='K') nc_add_variable(ds, val=data['SurfPres']['var'], dimension=('time',), var_name='SurfPres', type=np.float32, long_name='Surface atmospheric pressure from weather station', unit='hPa') nc_add_variable(ds, val=data['SurfWS']['var'], dimension=('time',), var_name='SurfWS', type=np.float32, long_name='Surface wind speed from weather station', unit='m/s') nc_add_variable(ds, val=data['SurfWD']['var'], dimension=('time',), var_name='SurfWD', type=np.float32, long_name='Surface wind direction from weather station', unit='deg') nc_add_variable(ds, val=data['rr']['var'], dimension=('time',), var_name='Rain', type=np.float32, long_name='Rain rate from weather station', unit='mm/h') ds.close() print('save calibrated to :: ', ds_name) return 0
def rpg_radar2nc_eurec4a(data, path, **kwargs): """ This routine generates a daily NetCDF4 file for the RPG 94 GHz FMCW radar 'LIMRAD94'. The variables and metadata are designed for the EUREC4A campaign. Args: data (dict): dictionary of larda containers path (string): path where the NetCDF file is stored """ dt_start = h.ts_to_dt(data['Ze']['ts'][0]) h.make_dir(path) site_name = kwargs['site'] if 'site' in kwargs else 'no-site' cn_version = kwargs['version'] if 'version' in kwargs else 'python' hc_version = kwargs['heave_corr_version'] if 'heave_corr_version' in kwargs else None for_aeris = kwargs['for_aeris'] if 'for_aeris' in kwargs else False dataset_version = 'v1.2' ds_name = f'{path}/eurec4a_{site_name}_cloudradar_{h.ts_to_dt(data["Ze"]["ts"][0]):%Y%m%d}_{dataset_version}.nc' ncvers = '4' with netCDF4.Dataset(ds_name, 'w', format=f'NETCDF{ncvers}') as ds: ds.Conventions = 'CF-1.8' ds.title = 'LIMRAD94 (SLDR) Doppler Cloud Radar, calibrated file' ds.campaign_id = 'EUREC4A' ds.platform_id = 'Meteor' ds.instrument_id = 'LIMRAD94' ds.version_id = dataset_version ds.location = data['Ze']['paraminfo']['location'] ds.system = data['Ze']['paraminfo']['system'] ds.version = f'Variable names and dimensions prepared for upload to Aeris data center. 1.1: updated metadata; ' \ f'1.2: added hydrometeor_mask' ds.institution = 'Leipzig Institute for Meteorology (LIM), Leipzig, Germany' ds.contact = '*****@*****.**' ds.source = '94 GHz Cloud Radar LIMRAD94\nRadar type: Frequency Modulated Continuous Wave,' \ '\nTransmitter power 1.5 W typical (solid state amplifier)' \ '\nAntenna Type: Bi-static Cassegrain with 500 mm aperture\nBeam width: 0.48deg FWHM' ds.reference = 'W Band Cloud Radar LIMRAD94' \ '\nDocumentation and User Manual provided by manufacturer RPG Radiometer Physics GmbH\n' \ 'Information about system also available at https://www.radiometer-physics.de/' ds.calibrations = f'remove Precip. ghost: {kwargs["ghost_echo_1"]}\n, ' \ f'remove curtain ghost: {kwargs["ghost_echo_2"]}\n' \ f'despeckle: {kwargs["despeckle"]}\n, ' \ f'number of standard deviations above noise: {kwargs["NF"]}\n' \ f'spectra heave corrected: {kwargs["heave_correction"]}\n' \ f'spectra heave corrected with version: {hc_version}\n' \ f'spectra dealiased: {kwargs["dealiasing"]}' ds.description = 'Concatenated data files of LIMRAD 94GHz - FMCW Radar, ' \ 'filters applied: ghost-echo, despeckle, use only main peak,\n' \ 'spectra corrected for heave motion of ship and then dealiased' ds.history = 'Created ' + time.ctime(time.time()) ds._FillValue = data['Ze']['paraminfo']['fill_value'] ds.day = dt_start.day ds.month = dt_start.month ds.year = dt_start.year Ze_str = 'Zh' if cn_version == 'python' else 'Ze' vel_str = 'v' if cn_version == 'python' else 'vm' width_str = 'width' if cn_version == 'python' else 'sigma' dim_tupel = ('time', 'range') if cn_version == 'python' else ('range', 'time') n_chirps = len(data['no_av']) ds.createDimension('chirp', n_chirps) ds.createDimension('time', data['Ze']['ts'].size) ds.createDimension('range', data['Ze']['rg'].size) if cn_version == 'matlab': for ivar in ['Ze', 'VEL', 'sw', 'ldr', 'kurt', 'skew']: data[ivar]['var'] = data[ivar]['var'].T # scalar variables nc_add_variable( ds, val=94.0, dimension=(), var_name='frequency', type=np.float32, long_name='Radar frequency', units='GHz' ) nc_add_variable( ds, val=256, dimension=(), var_name='Numfft', type=np.float32, long_name='Number of points in FFT', units='1' ) if not for_aeris: nc_add_variable( ds, val=np.mean(data['MaxVel']['var']), dimension=(), var_name='NyquistVelocity', type=np.float32, long_name='Mean (over all chirps) Unambiguous Doppler velocity (+/-)', units='m s-1' ) nc_add_variable( ds, val=data['Ze']['paraminfo']['altitude'], dimension=(), var_name='altitude', type=np.float32, long_name='Height of instrument above mean sea level', units='m' ) if 'version' in kwargs and cn_version == 'python': nc_add_variable( ds, val=data['no_av'], dimension=('chirp',), var_name='NumSpectraAveraged', type=np.float32, long_name='Number of spectral averages', units='1' ) # time and range variable # convert to time since start of year if cn_version == 'python': ts = np.subtract(data['Ze']['ts'], datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc).timestamp()) ts_str = 'seconds since 2020-01-01 00:00:00 UTC' ts_unit = f'seconds since 2020-01-01 00:00:00 +00:00 (UTC)' ts_calendar = 'standard' rg = data['Ze']['rg'] elif cn_version == 'matlab': ts = np.subtract(data['Ze']['ts'], datetime.datetime(2001, 1, 1, 0, 0, 0, tzinfo=timezone.utc).timestamp()) ts_str = 'Seconds since 1st January 2001 00:00 UTC' ts_unit = 'sec' rg = data['Ze']['rg'] else: raise ValueError('Wrong version selected! Change version to "matlab" or "python"!') nc_add_variable(ds, val=ts, dimension=('time',), var_name='time', type=np.float64, long_name=ts_str, units=ts_unit, calendar=ts_calendar) nc_add_variable(ds, val=rg, dimension=('range',), var_name='range', type=np.float32, long_name='Range from antenna to the centre of each range gate', units='m') if for_aeris: # time dependent latitude, longitude variables nc_add_variable(ds, val=data['lat'], dimension=('time',), var_name='latitude', type=np.float32, long_name='Latitude', units='degrees_north') nc_add_variable(ds, val=data['lon'], dimension=('time',), var_name='longitude', type=np.float32, long_name='Longitude', units='degrees_east') else: # static lat lon variables nc_add_variable( ds, val=data['Ze']['paraminfo']['coordinates'][0], dimension=(), var_name='latitude', type=np.float32, long_name='latitude', units='degrees_north' ) nc_add_variable( ds, val=data['Ze']['paraminfo']['coordinates'][1], dimension=(), var_name='longitude', type=np.float32, long_name='longitude', units='degrees_east' ) # chirp dependent variables if for_aeris: nc_add_variable(ds, val=data['MaxVel']['var'][0], dimension=('chirp',), var_name='Nyquist_velocity', type=np.float32, long_name='Unambiguous Doppler velocity (+/-)', units='m s-1') else: nc_add_variable(ds, val=data['MaxVel']['var'][0], dimension=('chirp',), var_name='DoppMax', type=np.float32, long_name='Unambiguous Doppler velocity (+/-)', units='m s-1') # index plus (1 to n) for Matlab indexing nc_add_variable(ds, val=data['rg_offsets'], dimension=('chirp',), var_name='range_offsets', type=np.int32, long_name='chirp sequences start index array in altitude layer array', units='1') if for_aeris: nc_add_variable(ds, val=data['time_shift_array'], dimension=('time', 'chirp'), var_name='time_shift', type=np.float32, long_name='Time shift between radar and ship calculated for each hour and chirp', units='s') # 1D variables dims_1d = ('time',) nc_add_variable(ds, val=data['bt']['var'], dimension=dims_1d, var_name='bt', type=np.float32, long_name='Direct detection brightness temperature at 89 GHz', units='K') nc_add_variable(ds, val=data['LWP']['var'], dimension=dims_1d, var_name='lwp', type=np.float32, long_name='Liquid water path', units='g m-2') nc_add_variable(ds, val=data['rr']['var'], dimension=dims_1d, var_name='rain', type=np.float32, long_name='Rain rate from weather station', units='mm h-1') nc_add_variable(ds, val=data['SurfRelHum']['var'], dimension=dims_1d, var_name='SurfRelHum', type=np.float32, long_name='Relative humidity from weather station', units='%') # 2D variables nc_add_variable(ds, val=data['Ze']['var'], dimension=dim_tupel, var_name=Ze_str, type=np.float32, long_name='Radar reflectivity factor', units='dBZ', plot_range=data['Ze']['var_lims'], plot_scale='linear', comment='Calibrated reflectivity. Calibration convention: in the absence of attenuation, ' 'a cloud at 273 K containing one million 100-micron droplets per cubic metre will ' 'have a reflectivity of 0 dBZ at all frequencies.') nc_add_variable(ds, val=data['VEL_roll']['var'], dimension=dim_tupel, plot_range=data['VEL_roll']['var_lims'], plot_scale='linear', var_name=vel_str, type=np.float32, long_name='Best estimate of averaged mean Doppler velocity', units='m s-1', unit_html='m s<sup>-1</sup>', comment='This parameter is the radial component of the velocity, with positive velocities are ' 'away from the radar (i.e. up) and negative velocities moving towards the radar ' '(i.e. down). It was corrected for the ships heave motion. ' 'A rolling average over 3 time steps has been applied to it.', folding_velocity=data['MaxVel']['var'].max()) if for_aeris: nc_add_variable(ds, val=data['VEL']['var'], dimension=dim_tupel, plot_range=data['VEL']['var_lims'], plot_scale='linear', var_name=f'v_no_rolling_mean_applied', type=np.float32, long_name='Heave corrected mean Doppler velocity', units='m s-1', unit_html='m s<sup>-1</sup>', comment='This parameter is the radial component of the velocity, with positive velocities ' 'are away from the radar (i.e. up) and negative velocities moving towards the ' 'radar (i.e. down). It was corrected for the ships heave motion but no rolling ' 'mean was applied.', folding_velocity=data['MaxVel']['var'].max()) nc_add_variable(ds, val=data['VEL_uncor']['var'], dimension=dim_tupel, plot_range=data['VEL_uncor']['var_lims'], plot_scale='linear', var_name='v_uncor', type=np.float32, long_name='Uncorrected Mean Doppler velocity', units='m s-1', unit_html='m s<sup>-1</sup>', comment='This parameter is the uncorrected radial component of the velocity, with positive ' 'velocities are away from the radar (i.e. up) and negative velocities moving ' 'towards the radar (i.e. down).', folding_velocity=data['MaxVel']['var'].max()) nc_add_variable(ds, val=data['sw']['var'], dimension=dim_tupel, plot_range=data['sw']['var_lims'], plot_scale='logarithmic', var_name=width_str, type=np.float32, long_name='Spectral width', units='m s-1', unit_html='m s<sup>-1</sup>', comment='This parameter is the standard deviation of the reflectivity-weighted velocities ' 'in the radar pulse volume.') nc_add_variable(ds, val=data['ldr']['var'], dimension=dim_tupel, plot_range=[-30.0, 0.0], var_name='ldr', type=np.float32, long_name='Linear depolarisation ratio', units='dB', comment='This parameter is the ratio of cross-polar to co-polar reflectivity.') nc_add_variable(ds, val=data['kurt']['var'], dimension=dim_tupel, plot_range=data['kurt']['var_lims'], var_name='kurt', type=np.float32, long_name='Kurtosis', units='1') nc_add_variable(ds, val=data['skew']['var'], dimension=dim_tupel, plot_range=data['skew']['var_lims'], var_name='Skew', type=np.float32, long_name='Skewness', units='1') if for_aeris: nc_add_variable(ds, val=data['heave_cor'], dimension=dim_tupel, plot_range=data['VEL']['var_lims'], plot_scale='linear', var_name='heave_cor', type=np.float32, long_name='Heave rate correction', units='m s-1', unit_html='m s<sup>-1</sup>', comment='This is the velocity by which the original Doppler spectrum was corrected by. ' 'The heave rate is subtracted from the Doppler velocity, meaning the spectra is ' 'shifted to the left for positive heave rates and to the right for negative heave ' 'rates.') nc_add_variable(ds, val=data['heave_cor_bins'], dimension=dim_tupel, plot_scale='linear', var_name='heave_cor_bins', type=np.int32, long_name='Heave rate correction in Doppler spectra bins', units='1', comment='This is the number of bins by which the original Doppler spectrum was shifted by. ' 'Positive values shift the spectrum to the left, while negative values shift the ' 'spectrum to the right.') nc_add_variable(ds, val=data['cloud_bases_tops'], dimension=dim_tupel, var_name='cloud_bases_tops', type=np.int32, long_name='Cloud Bases and Tops', units='1', comment='Cloud Base: -1, Cloud Top: 1, Else: 0') nc_add_variable(ds, val=data['hydrometeor_mask'], dimension=dim_tupel, var_name='hydrometeor_mask', type=np.int32, long_name='Hydrometeor Mask', units='1', comment='Radar signal: 1, no signal: 0') print('save calibrated to :: ', ds_name) return 0