def read_mira( filename, for_quicklooks=False, ql_res=5, field_names=None, additional_metadata=None, file_field_names=False, exclude_fields=None, include_fields=None, ): """ Read MIRA-35C NetCDF ingest data. Parameters ---------- filename : str Name of NetCDF file to read data from. for_quicklooks : bool, optional True if data is for quicklooks, averaging according to ql_res. ql_res : int, optional If for_quicklooks is true, data resolution in minutes. field_names : dict, optional Dictionary mapping field names in the file names to radar field names. Unlike other read functions, fields not in this dictionary or having a value of None are still included in the radar.fields dictionary, to exclude them use the `exclude_fields` parameter. Fields which are mapped by this dictionary will be renamed from key to value. additional_metadata : dict of dicts, optional This parameter is not used, it is included for uniformity. file_field_names : bool, optional True to force the use of the field names from the file in which case the `field_names` parameter is ignored. False will use to `field_names` parameter to rename fields. exclude_fields : list or None, optional List of fields to exclude from the radar object. This is applied after the `file_field_names` and `field_names` parameters. Set to None to include all fields specified by include_fields. include_fields : list or None, optional List of fields to include from the radar object. This is applied after the `file_field_names` and `field_names` parameters. Set to None to include all fields not specified by exclude_fields. Returns ------- radar : Radar Radar object. """ # create metadata retrieval object filemetadata = FileMetadata( "cfradial", field_names, additional_metadata, file_field_names, exclude_fields, include_fields, ) # read the data ncobj = netCDF4.Dataset(filename) ncvars = ncobj.variables # 4.1 Global attribute -> move to metadata dictionary metadata = dict([(k, getattr(ncobj, k)) for k in ncobj.ncattrs()]) metadata["n_gates_vary"] = "false" # 4.2 Dimensions # If averaging, increase temporal resolution with res if for_quicklooks: orig_res = round( float(ncobj.hrd[ncobj.hrd.find("AVE") + 4:ncobj.hrd.find("\nC")])) res = round(ql_res * 60 / orig_res) # 4.3 Global variable -> move to metadata dictionary if "volume_number" in ncvars: metadata["volume_number"] = int(ncvars["volume_number"][:]) else: metadata["volume_number"] = 0 global_vars = { "platform_type": "fixed", "instrument_type": "radar", "primary_axis": "axis_z", } # ignore time_* global variables, these are calculated from the time # variable when the file is written. for var, default_value in global_vars.items(): if var in ncvars: metadata[var] = str(netCDF4.chartostring(ncvars[var][:])) else: metadata[var] = default_value # 4.4 coordinate variables -> create attribute dictionaries time = cfradial._ncvar_to_dict(ncvars["time"]) time["units"] = "seconds since 1970-01-01 00:00:00" time["data"] = np.array(np.ma.MaskedArray.tolist(time["data"]), dtype="int64") _range = cfradial._ncvar_to_dict(ncvars["range"]) if for_quicklooks: time["data"] = time["data"][::res] # 4.5 Ray dimension variables # 4.6 Location variables -> create attribute dictionaries # the only difference in this section to cfradial.read_cfradial is the # minor variable name differences: # latitude -> lat # longitude -> lon # altitdue -> alt latitude = { "_FillValue": -9999.0, "data": np.ma.array( data=[-float(ncobj.Latitude[:-1])], mask=False, fill_value=1e20, dtype=np.float32, ), "long_name": "Latitude", "standard_name": "latitude", "units": "degree_N", "valid_max": 90.0, "valid_min": -90.0, } longitude = { "_FillValue": -9999.0, "data": np.ma.array( data=[-float(ncobj.Longitude[:-1])], mask=False, fill_value=1e20, dtype=np.float32, ), "long_name": "Longitude", "standard_name": "longitude", "units": "degree_E", "valid_max": 180.0, "valid_min": -180.0, } altitude = { "_FillValue": -9999.0, "data": np.ma.array( data=[float(ncobj.Altitude[:-1])], mask=False, fill_value=1e20, dtype=np.float32, ), "long_name": "Altitude", "standard_name": "altitude", "units": "m", } # 4.7 Sweep variables -> create atrribute dictionaries # this is the section that needed the most work since the initial NetCDF # file did not contain any sweep information sweep_number = filemetadata("sweep_number") sweep_number["data"] = np.array([0], dtype=np.int32) sweep_mode = filemetadata("sweep_mode") sweep_mode["data"] = np.array(["vertical_pointing"], dtype=np.str) fixed_angle = filemetadata("fixed_angle") fixed_angle["data"] = np.array([90.0], dtype=np.float32) sweep_start_ray_index = filemetadata("sweep_start_ray_index") sweep_start_ray_index["data"] = np.array([0], dtype=np.int32) sweep_end_ray_index = filemetadata("sweep_end_ray_index") sweep_end_ray_index["data"] = np.array([ncvars["time"].size - 1], dtype=np.int32) # first sweep mode determines scan_type # this module is specific to vertically-pointing data scan_type = "vpt" # 4.8 Sensor pointing variables -> create attribute dictionaries # this section also required some changes since the initial NetCDF did not # contain any sensor pointing variables azimuth = filemetadata("azimuth") azimuth["data"] = 0.0 * np.ones(ncvars["time"].size, dtype=np.float32) elevation = filemetadata("elevation") elevation["data"] = 90.0 * np.ones(ncvars["time"].size, dtype=np.float32) # 4.9 Moving platform geo-reference variables # 4.10 Moments field data variables -> field attribute dictionary # all variables with dimensions of 'time', 'range' are fields keys = [k for k, v in ncvars.items() if v.dimensions == ("time", "range")] fields = {} for key in keys: field_name = filemetadata.get_field_name(key) if field_name is None: if exclude_fields is not None and key in exclude_fields: continue if include_fields is not None and not key in include_fields: continue field_name = key fields[field_name] = cfradial._ncvar_to_dict(ncvars[key]) if for_quicklooks: fields[field_name]["data"] = block_reduce( fields[field_name]["data"], block_size=(res, 1), func=np.nanmean, cval=1e20, ) if field_name in ("SNRg", "SNR", "Ze", "Zg", "Z", "LDRg", "LDR"): fields[field_name]["data"] = 10 * np.log10( fields[field_name]["data"]) fields[field_name]["units"] = "dBZ" # 4.5 instrument_parameters sub-convention -> instrument_parameters dict # this section needed multiple changes and/or additions since the # instrument parameters were primarily located in the global attributes # this section is likely still incomplete omega = float(299792458 / ncvars["lambda"][:]) frequency = filemetadata("frequency") frequency["data"] = np.array([omega / 1e9], dtype=np.float32) prt_mode = filemetadata("prt_mode") prt_mode["data"] = np.array(["fixed"], dtype=np.str) prf = float(ncvars["prf"][:]) prt = filemetadata("prt") prt["data"] = (1.0 / prf) * np.ones(ncvars["time"].size, dtype=np.float32) v_nq = float(ncvars["NyquistVelocity"][:]) nyquist_velocity = filemetadata("nyquist_velocity") nyquist_velocity["data"] = ( v_nq * np.ones(ncvars["time"].size, dtype=np.float32), ) samples = int(ncvars["nave"][:]) n_samples = filemetadata("n_samples") n_samples["data"] = samples * np.ones(ncvars["time"].size, dtype=np.int32) # 4.6 radar_parameters sub-convention -> instrument_parameters dict # this section needed multiple changes and/or additions since the # radar instrument parameters were primarily located in the global # attributes # this section is likely still incomplete instrument_parameters = { "frequency": frequency, "prt_mode": prt_mode, "prt": prt, "nyquist_velocity": nyquist_velocity, "n_samples": n_samples, } # 4.7 lidar_parameters sub-convention -> skip # 4.8 radar_calibration sub-convention -> skip # 4.9 Getting Melting Layer Height data to a new file melthei = { "var": "MeltHei", "long_name": ncvars["MeltHei"].long_name, "units": ncvars["MeltHei"].units, "yrange": ncvars["MeltHei"].yrange, } melthei_det = { "var": "MeltHeiDet", "long_name": "Melting Layer Height detected from LDR", "units": ncvars["MeltHeiDet"].units, "yrange": ncvars["MeltHeiDet"].yrange, } melthei_db = { "var": "MeltHeiDB", "long_name": ncvars["MeltHeiDB"].long_name, "units": ncvars["MeltHeiDB"].units, "yrange": ncvars["MeltHeiDB"].yrange, } if for_quicklooks: melthei["data"] = block_reduce( ncvars["MeltHei"][:], block_size=(res, ), func=np.nanmean, cval=np.nanmean(ncvars["MeltHei"][:]), ) melthei_det["data"] = block_reduce( ncvars["MeltHeiDet"][:], block_size=(res, ), func=np.nanmean, cval=np.nanmean(ncvars["MeltHeiDet"][:]), ) melthei_db["data"] = block_reduce( ncvars["MeltHeiDB"][:], block_size=(res, ), func=np.nanmean, cval=np.nanmean(ncvars["MeltHeiDB"][:]), ) else: melthei["data"] = ncvars["MeltHei"][:] melthei_det["data"] = ncvars["MeltHeiDet"][:] melthei_db["data"] = ncvars["MeltHeiDB"][:] # close NetCDF object ncobj.close() return ( Radar( time, _range, fields, metadata, scan_type, latitude, longitude, altitude, sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index, sweep_end_ray_index, azimuth, elevation, instrument_parameters=instrument_parameters, ), (melthei, melthei_det, melthei_db), )
def read_rainbow_hdf5(fname): """ Ingest a Brazilian radar HDF5 file into Py-ART. Requires h5py. Parameters ---------- fname : str Name of Brazilian HDF5 radar file Returns ------- Radar : pyart.core.radar.Radar Py-ART Radar object, ready for processing, diplay, gridding, and writing to file """ # Field names field_names = { 'moment_0': 'corrected_reflectivity', 'moment_1': 'reflectivity', 'moment_2': 'velocity', 'moment_3': 'spectrum_width', 'moment_4': 'differential_reflectivity', 'moment_5': 'filtered_differential_phase', 'moment_6': 'differential_phase', 'moment_7': 'specific_differential_phase', 'moment_8': 'cross_correlation_ratio'} filemetadata = FileMetadata('cfradial', field_names, None, False, None) r = h5py.File(fname) pr = _initial_process(r) # fixed_angle fixed_angle = filemetadata('fixed_angle') fixed_angle['data'] = np.unique(pr['elevations']) # elevation elevation = filemetadata('elevation') elevation['data'] = pr['elevations'] # azimuth azimuth = filemetadata('azimuth') azimuth['data'] = pr['azimuths'] # sweep_number sweep_number = filemetadata('sweep_number') nsweeps = len(fixed_angle['data']) sweep_number['data'] = np.arange(nsweeps, dtype='int32') # sweep_mode sweep_mode = filemetadata('sweep_mode') scan_type = r['scan0']['what'].attrs['scan_type'][0].lower() if scan_type in ['ppi', 'rhi']: sweep_mode['data'] = np.array(nsweeps * ['manual_' + scan_type]) else: # Guessing that if not RHI or PPI, then a pointing scan sweep_mode['data'] = np.array(nsweeps * ['pointing']) # sweep_start_ray_index, sweep_end_ray_index sweep_start_ray_index = filemetadata('sweep_start_ray_index') sweep_end_ray_index = filemetadata('sweep_end_ray_index') ssri = [] ssre = [] for ang in fixed_angle['data']: index = np.where(pr['elevations'] == ang)[0] ssri.append(np.min(index)) ssre.append(np.max(index)) sweep_start_ray_index['data'] = np.array(ssri, dtype='int') sweep_end_ray_index['data'] = np.array(ssre, dtype='int') # radar location latitude = filemetadata('latitude') longitude = filemetadata('longitude') altitude = filemetadata('altitude') latitude['data'] = np.array(r['where'].attrs['lat']) longitude['data'] = np.array(r['where'].attrs['lon']) altitude['data'] = np.array(r['where'].attrs['height']) # time _time = filemetadata('time') start_time = pr['datetime'][0] _time['units'] = make_time_unit_str(start_time) _time['data'] = np.array( [(dti-start_time).total_seconds() for dti in pr['datetime']]) # range _range = filemetadata('range') _range['data'] = pr['range'] _range['meters_to_center_of_first_gate'] = _range['data'][0] / 2.0 _range['meters_between_gates'] = np.median(np.diff(_range['data'])) _range['spacing_is_constant'] = 1 # instrument_parameters instrument_parameters = {} for lab in ['nyquist_velocity', 'unambiguous_range']: tmpdic = filemetadata(lab) instrument_parameters[lab] = tmpdic tmpdic['data'] = pr[lab] # fields keys = [k for k in field_names.keys()] fields = {} for key in keys: field_name = filemetadata.get_field_name(key) fields[field_name] = {} fields[field_name]['data'] = pr['fields'][key] fields[field_name]['long_name'] = field_name fields[field_name]['units'] = r['scan0'][key].attrs['unit'][0] fields[field_name]['coordinates'] = 'elevation azimuth range' # metadata metadata = filemetadata('metadata') metadata['source'] = 'Brazil Radar' metadata['original_container'] = fname return Radar( _time, _range, fields, metadata, scan_type, latitude, longitude, altitude, sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index, sweep_end_ray_index, azimuth, elevation, instrument_parameters=instrument_parameters)
def read_sband_archive(filename, field_names=None, additional_metadata=None, file_field_names=False, exclude_fields=None, delay_field_loading=False, station=None, scans=None, linear_interp=True, **kwargs): """ Read a S band Archive file. Parameters ---------- filename : str Filename of S band Archive file. field_names : dict, optional Dictionary mapping S band moments to radar field names. If a data type found in the file does not appear in this dictionary or has a value of None it will not be placed in the radar.fields dictionary. A value of None, the default, will use the mapping defined in the metadata configuration file. additional_metadata : dict of dicts, optional Dictionary of dictionaries to retrieve metadata from during this read. This metadata is not used during any successive file reads unless explicitly included. A value of None, the default, will not introduct any addition metadata and the file specific or default metadata as specified by the metadata configuration file will be used. file_field_names : bool, optional True to use the S band field names for the field names. If this case the field_names parameter is ignored. The field dictionary will likely only have a 'data' key, unless the fields are defined in `additional_metadata`. exclude_fields : list or None, optional List of fields to exclude from the radar object. This is applied after the `file_field_names` and `field_names` parameters. delay_field_loading : bool, optional True to delay loading of field data from the file until the 'data' key in a particular field dictionary is accessed. In this case the field attribute of the returned Radar object will contain LazyLoadDict objects not dict objects. station : tuple or None, optional Three float tuple, include latitude, longitude and height of S band radar, first latitude, second longitude, third height (units: metre) scans : list or None, optional Read only specified scans from the file. None (the default) will read all scans. linear_interp : bool, optional True (the default) to perform linear interpolation between valid pairs of gates in low resolution rays in files mixed resolution rays. False will perform a nearest neighbor interpolation. This parameter is not used if the resolution of all rays in the file or requested sweeps is constant. Returns ------- radar : Radar Radar object containing all moments and sweeps/cuts in the volume. Gates not collected are masked in the field data. """ # test for non empty kwargs _test_arguments(kwargs) # create metadata retrieval object filemetadata = FileMetadata('nexrad_archive', field_names, additional_metadata, file_field_names, exclude_fields) # open the file and retrieve scan information nfile = SbandRadarFile(prepare_for_read(filename)) scan_info = nfile.scan_info(scans) # time time = filemetadata('time') time_start, _time = nfile.get_times(scans) time['data'] = _time time['units'] = make_time_unit_str(time_start) # range _range = filemetadata('range') first_gate, gate_spacing, last_gate = _find_range_params( scan_info, filemetadata) _range['data'] = np.arange(first_gate, last_gate, gate_spacing, 'float32') _range['meters_to_center_of_first_gate'] = float(first_gate) _range['meters_between_gates'] = float(gate_spacing) # metadata metadata = filemetadata('metadata') metadata['original_container'] = 'S band' vcp_pattern = nfile.get_vcp_pattern() if vcp_pattern is not None: metadata['vcp_pattern'] = vcp_pattern # scan_type scan_type = 'ppi' # latitude, longitude, altitude latitude = filemetadata('latitude') longitude = filemetadata('longitude') altitude = filemetadata('altitude') if station is None: lat, lon, alt = 0, 0, 0 else: lat, lon, alt = station latitude['data'] = np.array([lat], dtype='float64') longitude['data'] = np.array([lon], dtype='float64') altitude['data'] = np.array([alt], dtype='float64') # sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index # sweep_end_ray_index sweep_number = filemetadata('sweep_number') sweep_mode = filemetadata('sweep_mode') sweep_start_ray_index = filemetadata('sweep_start_ray_index') sweep_end_ray_index = filemetadata('sweep_end_ray_index') if scans is None: nsweeps = int(nfile.nscans) else: nsweeps = len(scans) sweep_number['data'] = np.arange(nsweeps, dtype='int32') sweep_mode['data'] = np.array(nsweeps * ['azimuth_surveillance'], dtype='S') rays_per_scan = [s['nrays'] for s in scan_info] sweep_end_ray_index['data'] = np.cumsum(rays_per_scan, dtype='int32') - 1 rays_per_scan.insert(0, 0) sweep_start_ray_index['data'] = np.cumsum(rays_per_scan[:-1], dtype='int32') # azimuth, elevation, fixed_angle azimuth = filemetadata('azimuth') elevation = filemetadata('elevation') fixed_angle = filemetadata('fixed_angle') azimuth['data'] = nfile.get_azimuth_angles(scans) elevation['data'] = nfile.get_elevation_angles(scans).astype('float32') fixed_angle['data'] = nfile.get_target_angles(scans) # fields max_ngates = len(_range['data']) available_moments = set([m for scan in scan_info for m in scan['moments']]) interpolate = _find_scans_to_interp(scan_info, first_gate, gate_spacing, filemetadata) fields = {} for moment in available_moments: field_name = filemetadata.get_field_name(moment) if field_name is None: continue dic = filemetadata(field_name) dic['_FillValue'] = get_fillvalue() if delay_field_loading and moment not in interpolate: dic = LazyLoadDict(dic) data_call = _NEXRADLevel2StagedField(nfile, moment, max_ngates, scans) dic.set_lazy('data', data_call) else: mdata = nfile.get_data(moment, max_ngates, scans=scans) if moment in interpolate: interp_scans = interpolate[moment] warnings.warn( "Gate spacing is not constant, interpolating data in " + "scans %s for moment %s." % (interp_scans, moment), UserWarning) for scan in interp_scans: idx = scan_info[scan]['moments'].index(moment) moment_ngates = scan_info[scan]['ngates'][idx] start = sweep_start_ray_index['data'][scan] end = sweep_end_ray_index['data'][scan] _interpolate_scan(mdata, start, end, moment_ngates, linear_interp) dic['data'] = mdata fields[field_name] = dic # instrument_parameters nyquist_velocity = filemetadata('nyquist_velocity') unambiguous_range = filemetadata('unambiguous_range') nyquist_velocity['data'] = nfile.get_nyquist_vel(scans).astype('float32') unambiguous_range['data'] = ( nfile.get_unambigous_range(scans).astype('float32')) instrument_parameters = { 'unambiguous_range': unambiguous_range, 'nyquist_velocity': nyquist_velocity, } nfile.close() return Radar(time, _range, fields, metadata, scan_type, latitude, longitude, altitude, sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index, sweep_end_ray_index, azimuth, elevation, instrument_parameters=instrument_parameters)
def read_uf(filename, field_names=None, additional_metadata=None, file_field_names=False, exclude_fields=None, delay_field_loading=False, **kwargs): """ Read a UF File. Parameters ---------- filename : str or file-like Name of Universal format file to read data from. field_names : dict, optional Dictionary mapping UF data type names to radar field names. If a data type found in the file does not appear in this dictionary or has a value of None it will not be placed in the radar.fields dictionary. A value of None, the default, will use the mapping defined in the Py-ART configuration file. additional_metadata : dict of dicts, optional Dictionary of dictionaries to retrieve metadata from during this read. This metadata is not used during any successive file reads unless explicitly included. A value of None, the default, will not introduct any addition metadata and the file specific or default metadata as specified by the Py-ART configuration file will be used. file_field_names : bool, optional True to force the use of the field names from the file in which case the `field_names` parameter is ignored. False will use to `field_names` parameter to rename fields. exclude_fields : list or None, optional List of fields to exclude from the radar object. This is applied after the `file_field_names` and `field_names` parameters. delay_field_loading : bool This option is not implemented in the function but included for compatability. Returns ------- radar : Radar Radar object. """ # test for non empty kwargs _test_arguments(kwargs) # create metadata retrieval object filemetadata = FileMetadata('uf', field_names, additional_metadata, file_field_names, exclude_fields) # Open UF file and get handle ufile = UFFile(filename) first_ray = ufile.rays[0] # time dts = ufile.get_datetimes() units = make_time_unit_str(min(dts)) time = filemetadata('time') time['units'] = units time['data'] = date2num(dts, units).astype('float32') # range _range = filemetadata('range') # assume that the number of gates and spacing from the first ray is # representative of the entire volume field_header = first_ray.field_headers[0] ngates = field_header['nbins'] start = field_header['range_start_m'] step = field_header['range_spacing_m'] # this gives distances to the start of each gate, add step/2 for center _range['data'] = np.arange(ngates, dtype='float32') * step + start _range['meters_to_center_of_first_gate'] = start _range['meters_between_gates'] = step # latitude, longitude and altitude latitude = filemetadata('latitude') longitude = filemetadata('longitude') altitude = filemetadata('altitude') lat, lon, height = first_ray.get_location() latitude['data'] = np.array([lat], dtype='float64') longitude['data'] = np.array([lon], dtype='float64') altitude['data'] = np.array([height], dtype='float64') # metadata metadata = filemetadata('metadata') metadata['original_container'] = 'UF' metadata['site_name'] = first_ray.mandatory_header['site_name'] metadata['radar_name'] = first_ray.mandatory_header['radar_name'] # sweep_start_ray_index, sweep_end_ray_index sweep_start_ray_index = filemetadata('sweep_start_ray_index') sweep_end_ray_index = filemetadata('sweep_end_ray_index') sweep_start_ray_index['data'] = ufile.first_ray_in_sweep sweep_end_ray_index['data'] = ufile.last_ray_in_sweep # sweep number sweep_number = filemetadata('sweep_number') sweep_number['data'] = np.arange(ufile.nsweeps, dtype='int32') # sweep_type scan_type = _UF_SWEEP_MODES[first_ray.mandatory_header['sweep_mode']] # sweep_mode sweep_mode = filemetadata('sweep_mode') sweep_mode['data'] = np.array(ufile.nsweeps * [_SWEEP_MODE_STR[scan_type]]) # elevation elevation = filemetadata('elevation') elevation['data'] = ufile.get_elevations() # azimuth azimuth = filemetadata('azimuth') azimuth['data'] = ufile.get_azimuths() # fixed_angle fixed_angle = filemetadata('fixed_angle') fixed_angle['data'] = ufile.get_sweep_fixed_angles() # fields fields = {} for uf_field_number, uf_field_dic in enumerate(first_ray.field_positions): uf_field_name = uf_field_dic['data_type'] field_name = filemetadata.get_field_name(uf_field_name) if field_name is None: continue field_dic = filemetadata(field_name) field_dic['data'] = ufile.get_field_data(uf_field_number) field_dic['_FillValue'] = get_fillvalue() fields[field_name] = field_dic # instrument_parameters instrument_parameters = _get_instrument_parameters(ufile, filemetadata) # scan rate scan_rate = filemetadata('scan_rate') scan_rate['data'] = ufile.get_sweep_rates() return Radar(time, _range, fields, metadata, scan_type, latitude, longitude, altitude, sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index, sweep_end_ray_index, azimuth, elevation, scan_rate=scan_rate, instrument_parameters=instrument_parameters)
def read_rainbow_hdf5(fname): """ Ingest a Brazilian radar HDF5 file into Py-ART. Requires h5py. Parameters ---------- fname : str Name of Brazilian HDF5 radar file Returns ------- Radar : pyart.core.radar.Radar Py-ART Radar object, ready for processing, diplay, gridding, and writing to file """ # Field names field_names = { "moment_0": "corrected_reflectivity", "moment_1": "reflectivity", "moment_2": "velocity", "moment_3": "spectrum_width", "moment_4": "differential_reflectivity", "moment_5": "filtered_differential_phase", "moment_6": "differential_phase", "moment_7": "specific_differential_phase", "moment_8": "cross_correlation_ratio", } filemetadata = FileMetadata("cfradial", field_names, None, False, None) r = h5py.File(fname) pr = _initial_process(r) # fixed_angle fixed_angle = filemetadata("fixed_angle") fixed_angle["data"] = np.unique(pr["elevations"]) # elevation elevation = filemetadata("elevation") elevation["data"] = pr["elevations"] # azimuth azimuth = filemetadata("azimuth") azimuth["data"] = pr["azimuths"] # sweep_number sweep_number = filemetadata("sweep_number") nsweeps = len(fixed_angle["data"]) sweep_number["data"] = np.arange(nsweeps, dtype="int32") # sweep_mode sweep_mode = filemetadata("sweep_mode") scan_type = str(r["scan0"]["what"].attrs["scan_type"][0])[2:-1].lower() if scan_type in ["ppi", "rhi"]: sweep_mode["data"] = np.array(nsweeps * ["manual_" + scan_type]) else: # Guessing that if not RHI or PPI, then a pointing scan sweep_mode["data"] = np.array(nsweeps * ["pointing"]) # sweep_start_ray_index, sweep_end_ray_index sweep_start_ray_index = filemetadata("sweep_start_ray_index") sweep_end_ray_index = filemetadata("sweep_end_ray_index") ssri = [] ssre = [] for ang in fixed_angle["data"]: index = np.where(pr["elevations"] == ang)[0] ssri.append(np.min(index)) ssre.append(np.max(index)) sweep_start_ray_index["data"] = np.array(ssri, dtype="int") sweep_end_ray_index["data"] = np.array(ssre, dtype="int") # radar location latitude = filemetadata("latitude") longitude = filemetadata("longitude") altitude = filemetadata("altitude") latitude["data"] = np.array(r["where"].attrs["lat"]) longitude["data"] = np.array(r["where"].attrs["lon"]) altitude["data"] = np.array(r["where"].attrs["height"]) # time _time = filemetadata("time") start_time = pr["datetime"][0] _time["units"] = make_time_unit_str(start_time) _time["data"] = np.array([(dti - start_time).total_seconds() for dti in pr["datetime"]]) # range _range = filemetadata("range") _range["data"] = pr["range"] _range["meters_to_center_of_first_gate"] = _range["data"][0] / 2.0 _range["meters_between_gates"] = np.median(np.diff(_range["data"])) _range["spacing_is_constant"] = 1 # instrument_parameters instrument_parameters = {} for lab in ["nyquist_velocity", "unambiguous_range"]: tmpdic = filemetadata(lab) instrument_parameters[lab] = tmpdic tmpdic["data"] = pr[lab] # fields keys = [k for k in field_names.keys()] fields = {} for key in keys: field_name = filemetadata.get_field_name(key) fields[field_name] = {} fields[field_name]["data"] = pr["fields"][key] fields[field_name]["long_name"] = field_name fields[field_name]["standard_name"] = field_name.replace("_", " ") fields[field_name]["units"] = str( r["scan0"][key].attrs["unit"][0])[2:-1] fields[field_name]["coordinates"] = "elevation azimuth range" # metadata metadata = filemetadata("metadata") metadata["source"] = "Brazil Radar" metadata["original_container"] = fname return Radar( _time, _range, fields, metadata, scan_type, latitude, longitude, altitude, sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index, sweep_end_ray_index, azimuth, elevation, instrument_parameters=instrument_parameters, )
def load_mrms_ppi(fdict, **kwargs): """ Read multiple field sweeps from an MRMS radar file NetCDF file. Input parameters ---------------- fdict : (list) --> list of dicts [{file: file1, ncvar: "Reflectivity", pvar: "reflecitity"}, {file: file2, ncvar: "Velocity", pvar: "corrected_velocity"}] filename : (str) --> name of netCDF MRMS file to read from ncvar : (str) --> name of variable to read from that file pvar : (str) --> mapped name of ncvar into pyART Returns ------- radar : Radar --> pyArt radar object TODO: For a given set of tilts, all the tilts will be set to the smallest number of gates in the tilts. The data structure is not quite correct. """ _debug = 0 # Loop over files to find the dimensions of the data. n_gates = [1192 ] # choose this to be the maximum number of gates we ever need. n_rays = [] n_elev = [] gates = [] for n, d in enumerate(fdict): try: ncfile = ncdf.Dataset(d['file']) except IOError: print('LOAD_PPI cannot open netCDF file: ', d['file']) break n_gates.append(len(ncfile.dimensions['Gate'])) n_rays.append(len(ncfile.dimensions['Azimuth'])) n_elev.append(ncfile.Elevation) ncfile.close() # important to do this. _mygate = min(n_gates) if _debug > 0: print(n_gates) print('LOAD_PPI --> Number of files to read: %d' % len(fdict)) print('LOAD_PPI --> _mygate: %d' % _mygate) # if we get this far, go ahead get to creating the radar object # this does not do anything yet except uses the object to create other objects # the default configuration is cfradial - for later. filemetadata = FileMetadata('cfradial') # create all the objects needed below _latitude = filemetadata('latitude') _longitude = filemetadata('longitude') _altitude = filemetadata('altitude') _metadata = filemetadata('metadata') _sweep_start_ray_index = filemetadata('sweep_start_ray_index') _sweep_end_ray_index = filemetadata('sweep_end_ray_index') _sweep_number = filemetadata('sweep_number') _sweep_mode = filemetadata('sweep_mode') _fixed_angle = filemetadata('fixed_angle') _time = filemetadata('time') _elevation = filemetadata('elevation') _azimuth = filemetadata('azimuth') _range = filemetadata('range') _scan_type = 'other' _fields = {} # dict to hold data # loop through files.. for n, d in enumerate(fdict): ncfile = ncdf.Dataset(d['file']) pvar = d['pvar'] ncvar = d['ncvar'] gwidth = ncfile.variables['GateWidth'][:].mean() if n == 0: # do these things once start_time = datetime.datetime.utcfromtimestamp(ncfile.Time) _time['data'] = np.array([ncfile.FractionalTime][:]) _time['units'] = make_time_unit_str(start_time) _latitude['data'] = np.array(ncfile.Latitude) _longitude['data'] = np.array(ncfile.Longitude) _altitude['data'] = np.array([ncfile.Height], 'float64') _range['data'] = ncfile.RangeToFirstGate + ncfile.variables['GateWidth'][0] \ * (np.arange(_mygate-1) + 0.5) _sweep_mode['data'] = np.array(['ppi']) _azimuth['data'] = np.array(ncfile.variables['Azimuth'][:]) _fixed_angle['data'] = np.array(n_elev) _elevation['data'] = np.array(n_rays[n] * [n_elev[n]]) _sweep_number['data'] = np.arange(len(fdict), dtype='int32') _sweep_start_ray_index['data'] = np.cumsum( np.append([0], n_rays[:-1]).astype('int32')) _sweep_end_ray_index['data'] = np.cumsum(n_rays).astype( 'int32') - 1 # copy meta data once metadata_mapping = { 'vcp-value': 'vcp-value', 'radarName-value': 'instrument_name', } for netcdf_attr, metadata_key in metadata_mapping.items(): if netcdf_attr in ncfile.ncattrs(): print(metadata_key, ncfile.getncattr(netcdf_attr)) _metadata[metadata_key] = ncfile.getncattr(netcdf_attr) # Okay do the big stuff. _dict = get_metadata(pvar) _dict['data'] = np.ma.array(ncfile.variables[ncvar][:, 0:_mygate - 1]) if 'MissingData' in ncfile.ncattrs(): _dict['data'][_dict['data'] == ncfile.MissingData] = np.ma.masked if 'RangeFolded' in ncfile.ncattrs(): _dict['data'][_dict['data'] == ncfile.RangeFolded] = np.ma.masked _dict['units'] = ncfile.getncattr('Unit-value') if _debug > 299: print(ncfile.variables[ncvar][:, 0:_mygate - 1].shape) print(_dict['data'].shape) _fields[pvar] = _dict # With elevation and azimuth in the radar object, lets recalculate # gate latitude, longitude and altitude, if _debug > 0: print('LOAD_PPI: Volume mean time: ', start_time) if _debug > 100: print('LOAD_PPI: final field dictionary: \n', _fields) print('LOAD_PPI: ngates.shape: ', _range['data'].shape) print('LOAD_PPI: nrays.shape: ', _azimuth['data'].shape) print('LOAD_PPI: sweep_start/stop: ', _sweep_start_ray_index['data'], _sweep_end_ray_index['data']) print('LOAD_PPI: sweeps: ', _sweep_number['data']) return Radar( _time, _range, _fields, _metadata, _scan_type, \ _latitude, _longitude, _altitude, \ _sweep_number, _sweep_mode, _fixed_angle, _sweep_start_ray_index, \ _sweep_end_ray_index, \ _azimuth, _elevation, instrument_parameters=None)
def c98dfile_archive(filename, field_names=None, additional_metadata=None, file_field_names=False, exclude_fields=None, cutnum=None, delay_field_loading=False, **kwargs): # test for non empty kwargs _test_arguments(kwargs) # create metadata retrieval object filemetadata = FileMetadata('c98d_archive', field_names, additional_metadata, file_field_names, exclude_fields) nfile = C98DRadFile(prepare_for_read(filename)) # scan_info = nfile.scan_info if cutnum is None: cutnum = nfile.scans else: cutnum = list(cutnum) # time time = filemetadata('time') _time = nfile.radial_info['seconds'] time['data'] = _time time['units'] = nfile.get_volume_start_time # range _range = filemetadata('range') _range['data'] = nfile.get_range('dBZ', cutnum) # _range['meters_to_center_of_first_gate'] = float(first_gate) # _range['meters_between_gates'] = float(gate_spacing) # fields fields = {} field_names = nfile.get_moment_type for field_name in field_names: dic = filemetadata(field_name) for i, cn in enumerate(cutnum): dic['_FillValue'] = get_fillvalue() if i == 0: dic['data'] = nfile.get_data(field_name, cn) else: fndata = nfile.get_data(field_name, cn) if fndata.shape[1] != dic['data'].shape[1]: if dic['data'].shape[1] - fndata.shape[1] > 0: apps = np.ones( (fndata.shape[0], dic['data'].shape[1] - fndata.shape[1])) * np.nan else: raise ValueError('something wrong!') fndata = np.c_[fndata, apps] dic['data'] = np.append(dic['data'], fndata, axis=0) fields.update({field_name: dic}) # scan_type scan_type = nfile.scan_type # latitude, longitude, altitude latitude = filemetadata('latitude') longitude = filemetadata('longitude') altitude = filemetadata('altitude') lat, lon, height = nfile.get_location latitude['data'] = np.array([lat], dtype='float64') longitude['data'] = np.array([lon], dtype='float64') altitude['data'] = np.array([height], dtype='float64') # sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index # sweep_end_ray_index sweep_number = filemetadata('sweep_number') sweep_mode = filemetadata('sweep_mode') sweep_start_ray_index = filemetadata('sweep_start_ray_index') sweep_end_ray_index = filemetadata('sweep_end_ray_index') sweep_number['data'] = np.arange(nfile.cutnum, dtype='int32') sweep_mode['data'] = np.array(nfile.cutnum * ['azimuth_surveillance'], dtype='S') sweep_end_ray_index['data'] = np.array(nfile.cut_end) sweep_start_ray_index['data'] = np.array(nfile.cut_start) # azimuth, elevation, fixed_angle azimuth = filemetadata('azimuth') elevation = filemetadata('elevation') fixed_angle = filemetadata('fixed_angle') azimuth['data'] = np.array(nfile.get_azimuth) elevation['data'] = np.array(nfile.get_elevation) fixed_angle['data'] = nfile.get_target_angles # instrument_parameters nyquist_velocity = filemetadata('nyquist_velocity') unambiguous_range = filemetadata('unambiguous_range') nyquist_velocity['data'] = None unambiguous_range['data'] = None instrument_parameters = { 'unambiguous_range': unambiguous_range, 'nyquist_velocity': nyquist_velocity } return Radar(time, _range, fields, filemetadata, scan_type, latitude, longitude, altitude, sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index, sweep_end_ray_index, azimuth, elevation, instrument_parameters=None)
def process_file(filename, outdir, debug=False, verbose=False): """ """ # Read radar data if USE_RADX: radar = read_radx(filename) else: radar = read(filename, exclude_fields=None) # Radar VCP check if CHECK_VCP: if NSWEEPS is not None and radar.nsweeps != NSWEEPS: return if verbose: print 'Processing file: {}'.format(os.path.basename(filename)) if debug: print 'Number of sweeps: {}'.format(radar.nsweeps) if USE_RADX: # Create file metadata object meta = FileMetadata( 'nexrad_archive', field_names=None, additional_metadata=None, file_field_names=False, exclude_fields=None) # Remove unnecessary fields for field in REMOVE_FIELDS: radar.fields.pop(field, None) # Rename fields to default Py-ART names for field in radar.fields.keys(): default_field = meta.get_field_name(field) radar.fields[default_field] = radar.fields.pop(field, None) # Step 1: Determine radar significant detection # Since NEXRAD WSR-88D Level II data is already processed to some degree, # this amounts to essentially removing salt and pepper noise gf = noise._significant_features( radar, REFL_FIELD, gatefilter=None, size_bins=SIZE_BINS, size_limits=SIZE_LIMITS, structure=STRUCTURE, remove_size_field=False, fill_value=None, size_field=None, debug=debug) gf = noise.significant_detection( radar, gatefilter=gf, remove_small_features=False, size_bins=SIZE_BINS, size_limits=SIZE_LIMITS, fill_holes=FILL_HOLES, dilate=DILATE, structure=STRUCTURE, iterations=1, rays_wrap_around=False, min_ncp=None, detect_field=None, debug=debug, verbose=verbose) # Step 2: Doppler velocity correction if DEALIAS == 'phase': vdop_corr = dealias_unwrap_phase( radar, gatefilter=gf, unwrap_unit='sweep', nyquist_vel=None, rays_wrap_around=True, keep_original=False, vel_field=None, corr_vel_field=VDOP_CORR_FIELD) elif DEALIAS == 'region': vdop_corr = dealias_region_based( radar, gatefilter=gf, interval_splits=INTERVAL_SPLITS, interval_limits=None, skip_between_rays=2, skip_along_ray=2, centered=True, nyquist_vel=None, rays_wrap_around=True, keep_original=False, vel_field=None, corr_vel_field=VDOP_CORR_FIELD) else: raise ValueError('Unsupported velocity correction routine') radar.add_field(VDOP_CORR_FIELD, vdop_corr, replace_existing=True) # Step 3: Reflectivity correction # Currently no correction procedures are applied to the reflectivity field # due to minimal attenuation at S-band refl_corr = radar.fields[REFL_FIELD].copy() radar.add_field(REFL_CORR_FIELD, refl_corr, replace_existing=True) # Step 4: Interpolate missing gates basic_fixes.interpolate_missing( radar, fields=FILL_FIELDS, interp_window=FILL_WINDOW, interp_sample=FILL_SAMPLE, kind='mean', rays_wrap_around=False, fill_value=None, debug=debug, verbose=verbose) # Add metadata _add_metadata(radar, filename) # ARM file name protocols date_stamp = datetimes_from_radar(radar).min().strftime('%Y%m%d.%H%M%S') fname = 'nexradwsr88d{}cmac{}.{}.{}.cdf'.format(QF, FN, DL, date_stamp) # Write CMAC NetCDF file write_cfradial(os.path.join(outdir, fname), radar, format=FORMAT, arm_time_variables=True) return
def read_casa_netcdf(filename, **kwargs): """ Read a CASA NetCDF file. Parameters ---------- filename : str Name of CASA NetCDF file to read data from. Returns ------- radar : Radar Radar object. """ # test for non empty kwargs _test_arguments(kwargs) # create metadata retrieval object filemetadata = FileMetadata('casa_netcdf') # Open netCDF4 file dset = netCDF4.Dataset(filename) nrays = len(dset.dimensions['Radial']) nbins = len(dset.dimensions['Gate']) # latitude, longitude and altitude latitude = filemetadata('latitude') longitude = filemetadata('longitude') altitude = filemetadata('altitude') latitude['data'] = np.array([dset.Latitude], 'float64') longitude['data'] = np.array([dset.Longitude], 'float64') altitude['data'] = np.array([dset.Height], 'float64') # metadata metadata = filemetadata('metadata') metadata_mapping = { 'vcp-value': 'vcp', #not in casa 'radarName-value': 'RadarName', 'ConversionPlugin': 'conversion_software', #not in casa } for netcdf_attr, metadata_key in metadata_mapping.items(): if netcdf_attr in dset.ncattrs(): metadata[metadata_key] = dset.getncattr(netcdf_attr) # sweep_start_ray_index, sweep_end_ray_index sweep_start_ray_index = filemetadata('sweep_start_ray_index') sweep_end_ray_index = filemetadata('sweep_end_ray_index') sweep_start_ray_index['data'] = np.array([0], dtype='int32') sweep_end_ray_index['data'] = np.array([nrays - 1], dtype='int32') # sweep number sweep_number = filemetadata('sweep_number') sweep_number['data'] = np.array([0], dtype='int32') # sweep_type scan_type = 'ppi' # sweep_mode, fixed_angle sweep_mode = filemetadata('sweep_mode') fixed_angle = filemetadata('fixed_angle') sweep_mode['data'] = np.array(1 * ['azimuth_surveillance']) elev_mean = np.mean(dset.variables['Elevation']) fixed_angle['data'] = np.array([elev_mean], dtype='float32') # time time = filemetadata('time') start_time = datetime.datetime.utcfromtimestamp(dset.variables['Time'][0]) time['units'] = make_time_unit_str(start_time) time['data'] = np.zeros((nrays, ), dtype='float64') # range _range = filemetadata('range') step = dset.variables['GateWidth'][0] / 1e3 # * len(dset.variables['GateWidth'][:]))/1e6 _range['data'] = (np.arange(nbins, dtype='float32') * step + step / 2) _range['meters_to_center_of_first_gate'] = step / 2. _range['meters_between_gates'] = step # elevation elevation = filemetadata('elevation') elevation_angle = elev_mean elevation['data'] = np.ones((nrays, ), dtype='float32') * elevation_angle # azimuth azimuth = filemetadata('azimuth') azimuth['data'] = dset.variables['Azimuth'][:] # fields # Ryan Note: What is "TypeName"? # field_name = dset.TypeName # field_data = np.ma.array(dset.variables[field_name][:]) # if 'MissingData' in dset.ncattrs(): # field_data[field_data == dset.MissingData] = np.ma.masked # if 'RangeFolded' in dset.ncattrs(): # field_data[field_data == dset.RangeFolded] = np.ma.masked # fields = {field_name: filemetadata(field_name)} # fields[field_name]['data'] = field_data # fields[field_name]['units'] = dset.variables[field_name].Units # fields[field_name]['_FillValue'] = get_fillvalue() # borrowing from d3r_gcpex_nc.py ncvars = dset.variables keys = [k for k, v in ncvars.items() if v.dimensions == ('Radial', 'Gate')] fields = {} for key in keys: field_name = filemetadata.get_field_name(key) if field_name is None: field_name = key fields[field_name] = _ncvar_to_dict(ncvars[key]) # instrument_parameters instrument_parameters = {} if 'PRF-value' in dset.ncattrs(): dic = filemetadata('prt') prt = 1. / float(dset.getncattr('PRF-value')) dic['data'] = np.ones((nrays, ), dtype='float32') * prt instrument_parameters['prt'] = dic if 'PulseWidth-value' in dset.ncattrs(): dic = filemetadata('pulse_width') pulse_width = dset.getncattr('PulseWidth-value') * 1.e-6 dic['data'] = np.ones((nrays, ), dtype='float32') * pulse_width instrument_parameters['pulse_width'] = dic if 'NyquistVelocity-value' in dset.ncattrs(): dic = filemetadata('nyquist_velocity') nyquist_velocity = float(dset.getncattr('NyquistVelocity-value')) dic['data'] = np.ones((nrays, ), dtype='float32') * nyquist_velocity instrument_parameters['nyquist_velocity'] = dic if 'Beamwidth' in dset.variables: dic = filemetadata('radar_beam_width_h') dic['data'] = dset.variables['Beamwidth'][:] instrument_parameters['radar_beam_width_h'] = dic dset.close() return Radar(time, _range, fields, metadata, scan_type, latitude, longitude, altitude, sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index, sweep_end_ray_index, azimuth, elevation, instrument_parameters=instrument_parameters)