def read_mira(
filename,
for_quicklooks=False,
ql_res=5,
field_names=None,
additional_metadata=None,
file_field_names=False,
exclude_fields=None,
include_fields=None,
):
"""
Read MIRA-35C NetCDF ingest data.
Parameters
----------
filename : str
Name of NetCDF file to read data from.
for_quicklooks : bool, optional
True if data is for quicklooks, averaging according to ql_res.
ql_res : int, optional
If for_quicklooks is true, data resolution in minutes.
field_names : dict, optional
Dictionary mapping field names in the file names to radar field names.
Unlike other read functions, fields not in this dictionary or having a
value of None are still included in the radar.fields dictionary, to
exclude them use the `exclude_fields` parameter. Fields which are
mapped by this dictionary will be renamed from key to value.
additional_metadata : dict of dicts, optional
This parameter is not used, it is included for uniformity.
file_field_names : bool, optional
True to force the use of the field names from the file in which
case the `field_names` parameter is ignored. False will use to
`field_names` parameter to rename fields.
exclude_fields : list or None, optional
List of fields to exclude from the radar object. This is applied
after the `file_field_names` and `field_names` parameters. Set
to None to include all fields specified by include_fields.
include_fields : list or None, optional
List of fields to include from the radar object. This is applied
after the `file_field_names` and `field_names` parameters. Set
to None to include all fields not specified by exclude_fields.
Returns
-------
radar : Radar
Radar object.
"""
# create metadata retrieval object
filemetadata = FileMetadata(
"cfradial",
field_names,
additional_metadata,
file_field_names,
exclude_fields,
include_fields,
)
# read the data
ncobj = netCDF4.Dataset(filename)
ncvars = ncobj.variables
# 4.1 Global attribute -> move to metadata dictionary
metadata = dict([(k, getattr(ncobj, k)) for k in ncobj.ncattrs()])
metadata["n_gates_vary"] = "false"
# 4.2 Dimensions
# If averaging, increase temporal resolution with res
if for_quicklooks:
orig_res = round(
float(ncobj.hrd[ncobj.hrd.find("AVE") + 4:ncobj.hrd.find("\nC")]))
res = round(ql_res * 60 / orig_res)
# 4.3 Global variable -> move to metadata dictionary
if "volume_number" in ncvars:
metadata["volume_number"] = int(ncvars["volume_number"][:])
else:
metadata["volume_number"] = 0
global_vars = {
"platform_type": "fixed",
"instrument_type": "radar",
"primary_axis": "axis_z",
}
# ignore time_* global variables, these are calculated from the time
# variable when the file is written.
for var, default_value in global_vars.items():
if var in ncvars:
metadata[var] = str(netCDF4.chartostring(ncvars[var][:]))
else:
metadata[var] = default_value
# 4.4 coordinate variables -> create attribute dictionaries
time = cfradial._ncvar_to_dict(ncvars["time"])
time["units"] = "seconds since 1970-01-01 00:00:00"
time["data"] = np.array(np.ma.MaskedArray.tolist(time["data"]),
dtype="int64")
_range = cfradial._ncvar_to_dict(ncvars["range"])
if for_quicklooks:
time["data"] = time["data"][::res]
# 4.5 Ray dimension variables
# 4.6 Location variables -> create attribute dictionaries
# the only difference in this section to cfradial.read_cfradial is the
# minor variable name differences:
# latitude -> lat
# longitude -> lon
# altitdue -> alt
latitude = {
"_FillValue":
-9999.0,
"data":
np.ma.array(
data=[-float(ncobj.Latitude[:-1])],
mask=False,
fill_value=1e20,
dtype=np.float32,
),
"long_name":
"Latitude",
"standard_name":
"latitude",
"units":
"degree_N",
"valid_max":
90.0,
"valid_min":
-90.0,
}
longitude = {
"_FillValue":
-9999.0,
"data":
np.ma.array(
data=[-float(ncobj.Longitude[:-1])],
mask=False,
fill_value=1e20,
dtype=np.float32,
),
"long_name":
"Longitude",
"standard_name":
"longitude",
"units":
"degree_E",
"valid_max":
180.0,
"valid_min":
-180.0,
}
altitude = {
"_FillValue":
-9999.0,
"data":
np.ma.array(
data=[float(ncobj.Altitude[:-1])],
mask=False,
fill_value=1e20,
dtype=np.float32,
),
"long_name":
"Altitude",
"standard_name":
"altitude",
"units":
"m",
}
# 4.7 Sweep variables -> create atrribute dictionaries
# this is the section that needed the most work since the initial NetCDF
# file did not contain any sweep information
sweep_number = filemetadata("sweep_number")
sweep_number["data"] = np.array([0], dtype=np.int32)
sweep_mode = filemetadata("sweep_mode")
sweep_mode["data"] = np.array(["vertical_pointing"], dtype=np.str)
fixed_angle = filemetadata("fixed_angle")
fixed_angle["data"] = np.array([90.0], dtype=np.float32)
sweep_start_ray_index = filemetadata("sweep_start_ray_index")
sweep_start_ray_index["data"] = np.array([0], dtype=np.int32)
sweep_end_ray_index = filemetadata("sweep_end_ray_index")
sweep_end_ray_index["data"] = np.array([ncvars["time"].size - 1],
dtype=np.int32)
# first sweep mode determines scan_type
# this module is specific to vertically-pointing data
scan_type = "vpt"
# 4.8 Sensor pointing variables -> create attribute dictionaries
# this section also required some changes since the initial NetCDF did not
# contain any sensor pointing variables
azimuth = filemetadata("azimuth")
azimuth["data"] = 0.0 * np.ones(ncvars["time"].size, dtype=np.float32)
elevation = filemetadata("elevation")
elevation["data"] = 90.0 * np.ones(ncvars["time"].size, dtype=np.float32)
# 4.9 Moving platform geo-reference variables
# 4.10 Moments field data variables -> field attribute dictionary
# all variables with dimensions of 'time', 'range' are fields
keys = [k for k, v in ncvars.items() if v.dimensions == ("time", "range")]
fields = {}
for key in keys:
field_name = filemetadata.get_field_name(key)
if field_name is None:
if exclude_fields is not None and key in exclude_fields:
continue
if include_fields is not None and not key in include_fields:
continue
field_name = key
fields[field_name] = cfradial._ncvar_to_dict(ncvars[key])
if for_quicklooks:
fields[field_name]["data"] = block_reduce(
fields[field_name]["data"],
block_size=(res, 1),
func=np.nanmean,
cval=1e20,
)
if field_name in ("SNRg", "SNR", "Ze", "Zg", "Z", "LDRg", "LDR"):
fields[field_name]["data"] = 10 * np.log10(
fields[field_name]["data"])
fields[field_name]["units"] = "dBZ"
# 4.5 instrument_parameters sub-convention -> instrument_parameters dict
# this section needed multiple changes and/or additions since the
# instrument parameters were primarily located in the global attributes
# this section is likely still incomplete
omega = float(299792458 / ncvars["lambda"][:])
frequency = filemetadata("frequency")
frequency["data"] = np.array([omega / 1e9], dtype=np.float32)
prt_mode = filemetadata("prt_mode")
prt_mode["data"] = np.array(["fixed"], dtype=np.str)
prf = float(ncvars["prf"][:])
prt = filemetadata("prt")
prt["data"] = (1.0 / prf) * np.ones(ncvars["time"].size, dtype=np.float32)
v_nq = float(ncvars["NyquistVelocity"][:])
nyquist_velocity = filemetadata("nyquist_velocity")
nyquist_velocity["data"] = (
v_nq * np.ones(ncvars["time"].size, dtype=np.float32), )
samples = int(ncvars["nave"][:])
n_samples = filemetadata("n_samples")
n_samples["data"] = samples * np.ones(ncvars["time"].size, dtype=np.int32)
# 4.6 radar_parameters sub-convention -> instrument_parameters dict
# this section needed multiple changes and/or additions since the
# radar instrument parameters were primarily located in the global
# attributes
# this section is likely still incomplete
instrument_parameters = {
"frequency": frequency,
"prt_mode": prt_mode,
"prt": prt,
"nyquist_velocity": nyquist_velocity,
"n_samples": n_samples,
}
# 4.7 lidar_parameters sub-convention -> skip
# 4.8 radar_calibration sub-convention -> skip
# 4.9 Getting Melting Layer Height data to a new file
melthei = {
"var": "MeltHei",
"long_name": ncvars["MeltHei"].long_name,
"units": ncvars["MeltHei"].units,
"yrange": ncvars["MeltHei"].yrange,
}
melthei_det = {
"var": "MeltHeiDet",
"long_name": "Melting Layer Height detected from LDR",
"units": ncvars["MeltHeiDet"].units,
"yrange": ncvars["MeltHeiDet"].yrange,
}
melthei_db = {
"var": "MeltHeiDB",
"long_name": ncvars["MeltHeiDB"].long_name,
"units": ncvars["MeltHeiDB"].units,
"yrange": ncvars["MeltHeiDB"].yrange,
}
if for_quicklooks:
melthei["data"] = block_reduce(
ncvars["MeltHei"][:],
block_size=(res, ),
func=np.nanmean,
cval=np.nanmean(ncvars["MeltHei"][:]),
)
melthei_det["data"] = block_reduce(
ncvars["MeltHeiDet"][:],
block_size=(res, ),
func=np.nanmean,
cval=np.nanmean(ncvars["MeltHeiDet"][:]),
)
melthei_db["data"] = block_reduce(
ncvars["MeltHeiDB"][:],
block_size=(res, ),
func=np.nanmean,
cval=np.nanmean(ncvars["MeltHeiDB"][:]),
)
else:
melthei["data"] = ncvars["MeltHei"][:]
melthei_det["data"] = ncvars["MeltHeiDet"][:]
melthei_db["data"] = ncvars["MeltHeiDB"][:]
# close NetCDF object
ncobj.close()
return (
Radar(
time,
_range,
fields,
metadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
instrument_parameters=instrument_parameters,
),
(melthei, melthei_det, melthei_db),
)
def read_rainbow_hdf5(fname):
"""
Ingest a Brazilian radar HDF5 file into Py-ART. Requires h5py.
Parameters
----------
fname : str
Name of Brazilian HDF5 radar file
Returns
-------
Radar : pyart.core.radar.Radar
Py-ART Radar object, ready for processing, diplay, gridding,
and writing to file
"""
# Field names
field_names = {
'moment_0': 'corrected_reflectivity',
'moment_1': 'reflectivity',
'moment_2': 'velocity',
'moment_3': 'spectrum_width',
'moment_4': 'differential_reflectivity',
'moment_5': 'filtered_differential_phase',
'moment_6': 'differential_phase',
'moment_7': 'specific_differential_phase',
'moment_8': 'cross_correlation_ratio'}
filemetadata = FileMetadata('cfradial', field_names, None,
False, None)
r = h5py.File(fname)
pr = _initial_process(r)
# fixed_angle
fixed_angle = filemetadata('fixed_angle')
fixed_angle['data'] = np.unique(pr['elevations'])
# elevation
elevation = filemetadata('elevation')
elevation['data'] = pr['elevations']
# azimuth
azimuth = filemetadata('azimuth')
azimuth['data'] = pr['azimuths']
# sweep_number
sweep_number = filemetadata('sweep_number')
nsweeps = len(fixed_angle['data'])
sweep_number['data'] = np.arange(nsweeps, dtype='int32')
# sweep_mode
sweep_mode = filemetadata('sweep_mode')
scan_type = r['scan0']['what'].attrs['scan_type'][0].lower()
if scan_type in ['ppi', 'rhi']:
sweep_mode['data'] = np.array(nsweeps * ['manual_' + scan_type])
else: # Guessing that if not RHI or PPI, then a pointing scan
sweep_mode['data'] = np.array(nsweeps * ['pointing'])
# sweep_start_ray_index, sweep_end_ray_index
sweep_start_ray_index = filemetadata('sweep_start_ray_index')
sweep_end_ray_index = filemetadata('sweep_end_ray_index')
ssri = []
ssre = []
for ang in fixed_angle['data']:
index = np.where(pr['elevations'] == ang)[0]
ssri.append(np.min(index))
ssre.append(np.max(index))
sweep_start_ray_index['data'] = np.array(ssri, dtype='int')
sweep_end_ray_index['data'] = np.array(ssre, dtype='int')
# radar location
latitude = filemetadata('latitude')
longitude = filemetadata('longitude')
altitude = filemetadata('altitude')
latitude['data'] = np.array(r['where'].attrs['lat'])
longitude['data'] = np.array(r['where'].attrs['lon'])
altitude['data'] = np.array(r['where'].attrs['height'])
# time
_time = filemetadata('time')
start_time = pr['datetime'][0]
_time['units'] = make_time_unit_str(start_time)
_time['data'] = np.array(
[(dti-start_time).total_seconds() for dti in pr['datetime']])
# range
_range = filemetadata('range')
_range['data'] = pr['range']
_range['meters_to_center_of_first_gate'] = _range['data'][0] / 2.0
_range['meters_between_gates'] = np.median(np.diff(_range['data']))
_range['spacing_is_constant'] = 1
# instrument_parameters
instrument_parameters = {}
for lab in ['nyquist_velocity', 'unambiguous_range']:
tmpdic = filemetadata(lab)
instrument_parameters[lab] = tmpdic
tmpdic['data'] = pr[lab]
# fields
keys = [k for k in field_names.keys()]
fields = {}
for key in keys:
field_name = filemetadata.get_field_name(key)
fields[field_name] = {}
fields[field_name]['data'] = pr['fields'][key]
fields[field_name]['long_name'] = field_name
fields[field_name]['units'] = r['scan0'][key].attrs['unit'][0]
fields[field_name]['coordinates'] = 'elevation azimuth range'
# metadata
metadata = filemetadata('metadata')
metadata['source'] = 'Brazil Radar'
metadata['original_container'] = fname
return Radar(
_time, _range, fields, metadata, scan_type,
latitude, longitude, altitude,
sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index,
sweep_end_ray_index,
azimuth, elevation,
instrument_parameters=instrument_parameters)
def read_sband_archive(filename,
field_names=None,
additional_metadata=None,
file_field_names=False,
exclude_fields=None,
delay_field_loading=False,
station=None,
scans=None,
linear_interp=True,
**kwargs):
"""
Read a S band Archive file.
Parameters
----------
filename : str
Filename of S band Archive file.
field_names : dict, optional
Dictionary mapping S band moments to radar field names. If a
data type found in the file does not appear in this dictionary or has
a value of None it will not be placed in the radar.fields dictionary.
A value of None, the default, will use the mapping defined in the
metadata configuration file.
additional_metadata : dict of dicts, optional
Dictionary of dictionaries to retrieve metadata from during this read.
This metadata is not used during any successive file reads unless
explicitly included. A value of None, the default, will not
introduct any addition metadata and the file specific or default
metadata as specified by the metadata configuration file will be used.
file_field_names : bool, optional
True to use the S band field names for the field names. If this
case the field_names parameter is ignored. The field dictionary will
likely only have a 'data' key, unless the fields are defined in
`additional_metadata`.
exclude_fields : list or None, optional
List of fields to exclude from the radar object. This is applied
after the `file_field_names` and `field_names` parameters.
delay_field_loading : bool, optional
True to delay loading of field data from the file until the 'data'
key in a particular field dictionary is accessed. In this case
the field attribute of the returned Radar object will contain
LazyLoadDict objects not dict objects.
station : tuple or None, optional
Three float tuple, include latitude, longitude and height of S band radar,
first latitude, second longitude, third height (units: metre)
scans : list or None, optional
Read only specified scans from the file. None (the default) will read
all scans.
linear_interp : bool, optional
True (the default) to perform linear interpolation between valid pairs
of gates in low resolution rays in files mixed resolution rays.
False will perform a nearest neighbor interpolation. This parameter is
not used if the resolution of all rays in the file or requested sweeps
is constant.
Returns
-------
radar : Radar
Radar object containing all moments and sweeps/cuts in the volume.
Gates not collected are masked in the field data.
"""
# test for non empty kwargs
_test_arguments(kwargs)
# create metadata retrieval object
filemetadata = FileMetadata('nexrad_archive', field_names,
additional_metadata, file_field_names,
exclude_fields)
# open the file and retrieve scan information
nfile = SbandRadarFile(prepare_for_read(filename))
scan_info = nfile.scan_info(scans)
# time
time = filemetadata('time')
time_start, _time = nfile.get_times(scans)
time['data'] = _time
time['units'] = make_time_unit_str(time_start)
# range
_range = filemetadata('range')
first_gate, gate_spacing, last_gate = _find_range_params(
scan_info, filemetadata)
_range['data'] = np.arange(first_gate, last_gate, gate_spacing, 'float32')
_range['meters_to_center_of_first_gate'] = float(first_gate)
_range['meters_between_gates'] = float(gate_spacing)
# metadata
metadata = filemetadata('metadata')
metadata['original_container'] = 'S band'
vcp_pattern = nfile.get_vcp_pattern()
if vcp_pattern is not None:
metadata['vcp_pattern'] = vcp_pattern
# scan_type
scan_type = 'ppi'
# latitude, longitude, altitude
latitude = filemetadata('latitude')
longitude = filemetadata('longitude')
altitude = filemetadata('altitude')
if station is None:
lat, lon, alt = 0, 0, 0
else:
lat, lon, alt = station
latitude['data'] = np.array([lat], dtype='float64')
longitude['data'] = np.array([lon], dtype='float64')
altitude['data'] = np.array([alt], dtype='float64')
# sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index
# sweep_end_ray_index
sweep_number = filemetadata('sweep_number')
sweep_mode = filemetadata('sweep_mode')
sweep_start_ray_index = filemetadata('sweep_start_ray_index')
sweep_end_ray_index = filemetadata('sweep_end_ray_index')
if scans is None:
nsweeps = int(nfile.nscans)
else:
nsweeps = len(scans)
sweep_number['data'] = np.arange(nsweeps, dtype='int32')
sweep_mode['data'] = np.array(nsweeps * ['azimuth_surveillance'],
dtype='S')
rays_per_scan = [s['nrays'] for s in scan_info]
sweep_end_ray_index['data'] = np.cumsum(rays_per_scan, dtype='int32') - 1
rays_per_scan.insert(0, 0)
sweep_start_ray_index['data'] = np.cumsum(rays_per_scan[:-1],
dtype='int32')
# azimuth, elevation, fixed_angle
azimuth = filemetadata('azimuth')
elevation = filemetadata('elevation')
fixed_angle = filemetadata('fixed_angle')
azimuth['data'] = nfile.get_azimuth_angles(scans)
elevation['data'] = nfile.get_elevation_angles(scans).astype('float32')
fixed_angle['data'] = nfile.get_target_angles(scans)
# fields
max_ngates = len(_range['data'])
available_moments = set([m for scan in scan_info for m in scan['moments']])
interpolate = _find_scans_to_interp(scan_info, first_gate, gate_spacing,
filemetadata)
fields = {}
for moment in available_moments:
field_name = filemetadata.get_field_name(moment)
if field_name is None:
continue
dic = filemetadata(field_name)
dic['_FillValue'] = get_fillvalue()
if delay_field_loading and moment not in interpolate:
dic = LazyLoadDict(dic)
data_call = _NEXRADLevel2StagedField(nfile, moment, max_ngates,
scans)
dic.set_lazy('data', data_call)
else:
mdata = nfile.get_data(moment, max_ngates, scans=scans)
if moment in interpolate:
interp_scans = interpolate[moment]
warnings.warn(
"Gate spacing is not constant, interpolating data in " +
"scans %s for moment %s." % (interp_scans, moment),
UserWarning)
for scan in interp_scans:
idx = scan_info[scan]['moments'].index(moment)
moment_ngates = scan_info[scan]['ngates'][idx]
start = sweep_start_ray_index['data'][scan]
end = sweep_end_ray_index['data'][scan]
_interpolate_scan(mdata, start, end, moment_ngates,
linear_interp)
dic['data'] = mdata
fields[field_name] = dic
# instrument_parameters
nyquist_velocity = filemetadata('nyquist_velocity')
unambiguous_range = filemetadata('unambiguous_range')
nyquist_velocity['data'] = nfile.get_nyquist_vel(scans).astype('float32')
unambiguous_range['data'] = (
nfile.get_unambigous_range(scans).astype('float32'))
instrument_parameters = {
'unambiguous_range': unambiguous_range,
'nyquist_velocity': nyquist_velocity,
}
nfile.close()
return Radar(time,
_range,
fields,
metadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
instrument_parameters=instrument_parameters)
def read_uf(filename,
field_names=None,
additional_metadata=None,
file_field_names=False,
exclude_fields=None,
delay_field_loading=False,
**kwargs):
"""
Read a UF File.
Parameters
----------
filename : str or file-like
Name of Universal format file to read data from.
field_names : dict, optional
Dictionary mapping UF data type names to radar field names. If a
data type found in the file does not appear in this dictionary or has
a value of None it will not be placed in the radar.fields dictionary.
A value of None, the default, will use the mapping defined in the
Py-ART configuration file.
additional_metadata : dict of dicts, optional
Dictionary of dictionaries to retrieve metadata from during this read.
This metadata is not used during any successive file reads unless
explicitly included. A value of None, the default, will not
introduct any addition metadata and the file specific or default
metadata as specified by the Py-ART configuration file will be used.
file_field_names : bool, optional
True to force the use of the field names from the file in which
case the `field_names` parameter is ignored. False will use to
`field_names` parameter to rename fields.
exclude_fields : list or None, optional
List of fields to exclude from the radar object. This is applied
after the `file_field_names` and `field_names` parameters.
delay_field_loading : bool
This option is not implemented in the function but included for
compatability.
Returns
-------
radar : Radar
Radar object.
"""
# test for non empty kwargs
_test_arguments(kwargs)
# create metadata retrieval object
filemetadata = FileMetadata('uf', field_names, additional_metadata,
file_field_names, exclude_fields)
# Open UF file and get handle
ufile = UFFile(filename)
first_ray = ufile.rays[0]
# time
dts = ufile.get_datetimes()
units = make_time_unit_str(min(dts))
time = filemetadata('time')
time['units'] = units
time['data'] = date2num(dts, units).astype('float32')
# range
_range = filemetadata('range')
# assume that the number of gates and spacing from the first ray is
# representative of the entire volume
field_header = first_ray.field_headers[0]
ngates = field_header['nbins']
start = field_header['range_start_m']
step = field_header['range_spacing_m']
# this gives distances to the start of each gate, add step/2 for center
_range['data'] = np.arange(ngates, dtype='float32') * step + start
_range['meters_to_center_of_first_gate'] = start
_range['meters_between_gates'] = step
# latitude, longitude and altitude
latitude = filemetadata('latitude')
longitude = filemetadata('longitude')
altitude = filemetadata('altitude')
lat, lon, height = first_ray.get_location()
latitude['data'] = np.array([lat], dtype='float64')
longitude['data'] = np.array([lon], dtype='float64')
altitude['data'] = np.array([height], dtype='float64')
# metadata
metadata = filemetadata('metadata')
metadata['original_container'] = 'UF'
metadata['site_name'] = first_ray.mandatory_header['site_name']
metadata['radar_name'] = first_ray.mandatory_header['radar_name']
# sweep_start_ray_index, sweep_end_ray_index
sweep_start_ray_index = filemetadata('sweep_start_ray_index')
sweep_end_ray_index = filemetadata('sweep_end_ray_index')
sweep_start_ray_index['data'] = ufile.first_ray_in_sweep
sweep_end_ray_index['data'] = ufile.last_ray_in_sweep
# sweep number
sweep_number = filemetadata('sweep_number')
sweep_number['data'] = np.arange(ufile.nsweeps, dtype='int32')
# sweep_type
scan_type = _UF_SWEEP_MODES[first_ray.mandatory_header['sweep_mode']]
# sweep_mode
sweep_mode = filemetadata('sweep_mode')
sweep_mode['data'] = np.array(ufile.nsweeps * [_SWEEP_MODE_STR[scan_type]])
# elevation
elevation = filemetadata('elevation')
elevation['data'] = ufile.get_elevations()
# azimuth
azimuth = filemetadata('azimuth')
azimuth['data'] = ufile.get_azimuths()
# fixed_angle
fixed_angle = filemetadata('fixed_angle')
fixed_angle['data'] = ufile.get_sweep_fixed_angles()
# fields
fields = {}
for uf_field_number, uf_field_dic in enumerate(first_ray.field_positions):
uf_field_name = uf_field_dic['data_type']
field_name = filemetadata.get_field_name(uf_field_name)
if field_name is None:
continue
field_dic = filemetadata(field_name)
field_dic['data'] = ufile.get_field_data(uf_field_number)
field_dic['_FillValue'] = get_fillvalue()
fields[field_name] = field_dic
# instrument_parameters
instrument_parameters = _get_instrument_parameters(ufile, filemetadata)
# scan rate
scan_rate = filemetadata('scan_rate')
scan_rate['data'] = ufile.get_sweep_rates()
return Radar(time,
_range,
fields,
metadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
scan_rate=scan_rate,
instrument_parameters=instrument_parameters)
def read_rainbow_hdf5(fname):
"""
Ingest a Brazilian radar HDF5 file into Py-ART. Requires h5py.
Parameters
----------
fname : str
Name of Brazilian HDF5 radar file
Returns
-------
Radar : pyart.core.radar.Radar
Py-ART Radar object, ready for processing, diplay, gridding,
and writing to file
"""
# Field names
field_names = {
"moment_0": "corrected_reflectivity",
"moment_1": "reflectivity",
"moment_2": "velocity",
"moment_3": "spectrum_width",
"moment_4": "differential_reflectivity",
"moment_5": "filtered_differential_phase",
"moment_6": "differential_phase",
"moment_7": "specific_differential_phase",
"moment_8": "cross_correlation_ratio",
}
filemetadata = FileMetadata("cfradial", field_names, None, False, None)
r = h5py.File(fname)
pr = _initial_process(r)
# fixed_angle
fixed_angle = filemetadata("fixed_angle")
fixed_angle["data"] = np.unique(pr["elevations"])
# elevation
elevation = filemetadata("elevation")
elevation["data"] = pr["elevations"]
# azimuth
azimuth = filemetadata("azimuth")
azimuth["data"] = pr["azimuths"]
# sweep_number
sweep_number = filemetadata("sweep_number")
nsweeps = len(fixed_angle["data"])
sweep_number["data"] = np.arange(nsweeps, dtype="int32")
# sweep_mode
sweep_mode = filemetadata("sweep_mode")
scan_type = str(r["scan0"]["what"].attrs["scan_type"][0])[2:-1].lower()
if scan_type in ["ppi", "rhi"]:
sweep_mode["data"] = np.array(nsweeps * ["manual_" + scan_type])
else: # Guessing that if not RHI or PPI, then a pointing scan
sweep_mode["data"] = np.array(nsweeps * ["pointing"])
# sweep_start_ray_index, sweep_end_ray_index
sweep_start_ray_index = filemetadata("sweep_start_ray_index")
sweep_end_ray_index = filemetadata("sweep_end_ray_index")
ssri = []
ssre = []
for ang in fixed_angle["data"]:
index = np.where(pr["elevations"] == ang)[0]
ssri.append(np.min(index))
ssre.append(np.max(index))
sweep_start_ray_index["data"] = np.array(ssri, dtype="int")
sweep_end_ray_index["data"] = np.array(ssre, dtype="int")
# radar location
latitude = filemetadata("latitude")
longitude = filemetadata("longitude")
altitude = filemetadata("altitude")
latitude["data"] = np.array(r["where"].attrs["lat"])
longitude["data"] = np.array(r["where"].attrs["lon"])
altitude["data"] = np.array(r["where"].attrs["height"])
# time
_time = filemetadata("time")
start_time = pr["datetime"][0]
_time["units"] = make_time_unit_str(start_time)
_time["data"] = np.array([(dti - start_time).total_seconds()
for dti in pr["datetime"]])
# range
_range = filemetadata("range")
_range["data"] = pr["range"]
_range["meters_to_center_of_first_gate"] = _range["data"][0] / 2.0
_range["meters_between_gates"] = np.median(np.diff(_range["data"]))
_range["spacing_is_constant"] = 1
# instrument_parameters
instrument_parameters = {}
for lab in ["nyquist_velocity", "unambiguous_range"]:
tmpdic = filemetadata(lab)
instrument_parameters[lab] = tmpdic
tmpdic["data"] = pr[lab]
# fields
keys = [k for k in field_names.keys()]
fields = {}
for key in keys:
field_name = filemetadata.get_field_name(key)
fields[field_name] = {}
fields[field_name]["data"] = pr["fields"][key]
fields[field_name]["long_name"] = field_name
fields[field_name]["standard_name"] = field_name.replace("_", " ")
fields[field_name]["units"] = str(
r["scan0"][key].attrs["unit"][0])[2:-1]
fields[field_name]["coordinates"] = "elevation azimuth range"
# metadata
metadata = filemetadata("metadata")
metadata["source"] = "Brazil Radar"
metadata["original_container"] = fname
return Radar(
_time,
_range,
fields,
metadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
instrument_parameters=instrument_parameters,
)
def load_mrms_ppi(fdict, **kwargs):
"""
Read multiple field sweeps from an MRMS radar file NetCDF file.
Input parameters
----------------
fdict : (list) --> list of dicts [{file: file1, ncvar: "Reflectivity", pvar: "reflecitity"},
{file: file2, ncvar: "Velocity", pvar: "corrected_velocity"}]
filename : (str) --> name of netCDF MRMS file to read from
ncvar : (str) --> name of variable to read from that file
pvar : (str) --> mapped name of ncvar into pyART
Returns
-------
radar : Radar --> pyArt radar object
TODO: For a given set of tilts, all the tilts will be set to the smallest number of gates in the
tilts. The data structure is not quite correct.
"""
_debug = 0
# Loop over files to find the dimensions of the data.
n_gates = [1192
] # choose this to be the maximum number of gates we ever need.
n_rays = []
n_elev = []
gates = []
for n, d in enumerate(fdict):
try:
ncfile = ncdf.Dataset(d['file'])
except IOError:
print('LOAD_PPI cannot open netCDF file: ', d['file'])
break
n_gates.append(len(ncfile.dimensions['Gate']))
n_rays.append(len(ncfile.dimensions['Azimuth']))
n_elev.append(ncfile.Elevation)
ncfile.close() # important to do this.
_mygate = min(n_gates)
if _debug > 0:
print(n_gates)
print('LOAD_PPI --> Number of files to read: %d' % len(fdict))
print('LOAD_PPI --> _mygate: %d' % _mygate)
# if we get this far, go ahead get to creating the radar object
# this does not do anything yet except uses the object to create other objects
# the default configuration is cfradial - for later.
filemetadata = FileMetadata('cfradial')
# create all the objects needed below
_latitude = filemetadata('latitude')
_longitude = filemetadata('longitude')
_altitude = filemetadata('altitude')
_metadata = filemetadata('metadata')
_sweep_start_ray_index = filemetadata('sweep_start_ray_index')
_sweep_end_ray_index = filemetadata('sweep_end_ray_index')
_sweep_number = filemetadata('sweep_number')
_sweep_mode = filemetadata('sweep_mode')
_fixed_angle = filemetadata('fixed_angle')
_time = filemetadata('time')
_elevation = filemetadata('elevation')
_azimuth = filemetadata('azimuth')
_range = filemetadata('range')
_scan_type = 'other'
_fields = {} # dict to hold data
# loop through files..
for n, d in enumerate(fdict):
ncfile = ncdf.Dataset(d['file'])
pvar = d['pvar']
ncvar = d['ncvar']
gwidth = ncfile.variables['GateWidth'][:].mean()
if n == 0: # do these things once
start_time = datetime.datetime.utcfromtimestamp(ncfile.Time)
_time['data'] = np.array([ncfile.FractionalTime][:])
_time['units'] = make_time_unit_str(start_time)
_latitude['data'] = np.array(ncfile.Latitude)
_longitude['data'] = np.array(ncfile.Longitude)
_altitude['data'] = np.array([ncfile.Height], 'float64')
_range['data'] = ncfile.RangeToFirstGate + ncfile.variables['GateWidth'][0] \
* (np.arange(_mygate-1) + 0.5)
_sweep_mode['data'] = np.array(['ppi'])
_azimuth['data'] = np.array(ncfile.variables['Azimuth'][:])
_fixed_angle['data'] = np.array(n_elev)
_elevation['data'] = np.array(n_rays[n] * [n_elev[n]])
_sweep_number['data'] = np.arange(len(fdict), dtype='int32')
_sweep_start_ray_index['data'] = np.cumsum(
np.append([0], n_rays[:-1]).astype('int32'))
_sweep_end_ray_index['data'] = np.cumsum(n_rays).astype(
'int32') - 1
# copy meta data once
metadata_mapping = {
'vcp-value': 'vcp-value',
'radarName-value': 'instrument_name',
}
for netcdf_attr, metadata_key in metadata_mapping.items():
if netcdf_attr in ncfile.ncattrs():
print(metadata_key, ncfile.getncattr(netcdf_attr))
_metadata[metadata_key] = ncfile.getncattr(netcdf_attr)
# Okay do the big stuff.
_dict = get_metadata(pvar)
_dict['data'] = np.ma.array(ncfile.variables[ncvar][:, 0:_mygate - 1])
if 'MissingData' in ncfile.ncattrs():
_dict['data'][_dict['data'] == ncfile.MissingData] = np.ma.masked
if 'RangeFolded' in ncfile.ncattrs():
_dict['data'][_dict['data'] == ncfile.RangeFolded] = np.ma.masked
_dict['units'] = ncfile.getncattr('Unit-value')
if _debug > 299:
print(ncfile.variables[ncvar][:, 0:_mygate - 1].shape)
print(_dict['data'].shape)
_fields[pvar] = _dict
# With elevation and azimuth in the radar object, lets recalculate
# gate latitude, longitude and altitude,
if _debug > 0:
print('LOAD_PPI: Volume mean time: ', start_time)
if _debug > 100:
print('LOAD_PPI: final field dictionary: \n', _fields)
print('LOAD_PPI: ngates.shape: ', _range['data'].shape)
print('LOAD_PPI: nrays.shape: ', _azimuth['data'].shape)
print('LOAD_PPI: sweep_start/stop: ', _sweep_start_ray_index['data'],
_sweep_end_ray_index['data'])
print('LOAD_PPI: sweeps: ', _sweep_number['data'])
return Radar( _time, _range, _fields, _metadata, _scan_type, \
_latitude, _longitude, _altitude, \
_sweep_number, _sweep_mode, _fixed_angle, _sweep_start_ray_index, \
_sweep_end_ray_index, \
_azimuth, _elevation, instrument_parameters=None)
def c98dfile_archive(filename,
field_names=None,
additional_metadata=None,
file_field_names=False,
exclude_fields=None,
cutnum=None,
delay_field_loading=False,
**kwargs):
# test for non empty kwargs
_test_arguments(kwargs)
# create metadata retrieval object
filemetadata = FileMetadata('c98d_archive', field_names,
additional_metadata, file_field_names,
exclude_fields)
nfile = C98DRadFile(prepare_for_read(filename))
# scan_info = nfile.scan_info
if cutnum is None:
cutnum = nfile.scans
else:
cutnum = list(cutnum)
# time
time = filemetadata('time')
_time = nfile.radial_info['seconds']
time['data'] = _time
time['units'] = nfile.get_volume_start_time
# range
_range = filemetadata('range')
_range['data'] = nfile.get_range('dBZ', cutnum)
# _range['meters_to_center_of_first_gate'] = float(first_gate)
# _range['meters_between_gates'] = float(gate_spacing)
# fields
fields = {}
field_names = nfile.get_moment_type
for field_name in field_names:
dic = filemetadata(field_name)
for i, cn in enumerate(cutnum):
dic['_FillValue'] = get_fillvalue()
if i == 0:
dic['data'] = nfile.get_data(field_name, cn)
else:
fndata = nfile.get_data(field_name, cn)
if fndata.shape[1] != dic['data'].shape[1]:
if dic['data'].shape[1] - fndata.shape[1] > 0:
apps = np.ones(
(fndata.shape[0],
dic['data'].shape[1] - fndata.shape[1])) * np.nan
else:
raise ValueError('something wrong!')
fndata = np.c_[fndata, apps]
dic['data'] = np.append(dic['data'], fndata, axis=0)
fields.update({field_name: dic})
# scan_type
scan_type = nfile.scan_type
# latitude, longitude, altitude
latitude = filemetadata('latitude')
longitude = filemetadata('longitude')
altitude = filemetadata('altitude')
lat, lon, height = nfile.get_location
latitude['data'] = np.array([lat], dtype='float64')
longitude['data'] = np.array([lon], dtype='float64')
altitude['data'] = np.array([height], dtype='float64')
# sweep_number, sweep_mode, fixed_angle, sweep_start_ray_index
# sweep_end_ray_index
sweep_number = filemetadata('sweep_number')
sweep_mode = filemetadata('sweep_mode')
sweep_start_ray_index = filemetadata('sweep_start_ray_index')
sweep_end_ray_index = filemetadata('sweep_end_ray_index')
sweep_number['data'] = np.arange(nfile.cutnum, dtype='int32')
sweep_mode['data'] = np.array(nfile.cutnum * ['azimuth_surveillance'],
dtype='S')
sweep_end_ray_index['data'] = np.array(nfile.cut_end)
sweep_start_ray_index['data'] = np.array(nfile.cut_start)
# azimuth, elevation, fixed_angle
azimuth = filemetadata('azimuth')
elevation = filemetadata('elevation')
fixed_angle = filemetadata('fixed_angle')
azimuth['data'] = np.array(nfile.get_azimuth)
elevation['data'] = np.array(nfile.get_elevation)
fixed_angle['data'] = nfile.get_target_angles
# instrument_parameters
nyquist_velocity = filemetadata('nyquist_velocity')
unambiguous_range = filemetadata('unambiguous_range')
nyquist_velocity['data'] = None
unambiguous_range['data'] = None
instrument_parameters = {
'unambiguous_range': unambiguous_range,
'nyquist_velocity': nyquist_velocity
}
return Radar(time,
_range,
fields,
filemetadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
instrument_parameters=None)
def process_file(filename, outdir, debug=False, verbose=False):
"""
"""
# Read radar data
if USE_RADX:
radar = read_radx(filename)
else:
radar = read(filename, exclude_fields=None)
# Radar VCP check
if CHECK_VCP:
if NSWEEPS is not None and radar.nsweeps != NSWEEPS:
return
if verbose:
print 'Processing file: {}'.format(os.path.basename(filename))
if debug:
print 'Number of sweeps: {}'.format(radar.nsweeps)
if USE_RADX:
# Create file metadata object
meta = FileMetadata(
'nexrad_archive', field_names=None, additional_metadata=None,
file_field_names=False, exclude_fields=None)
# Remove unnecessary fields
for field in REMOVE_FIELDS:
radar.fields.pop(field, None)
# Rename fields to default Py-ART names
for field in radar.fields.keys():
default_field = meta.get_field_name(field)
radar.fields[default_field] = radar.fields.pop(field, None)
# Step 1: Determine radar significant detection
# Since NEXRAD WSR-88D Level II data is already processed to some degree,
# this amounts to essentially removing salt and pepper noise
gf = noise._significant_features(
radar, REFL_FIELD, gatefilter=None, size_bins=SIZE_BINS,
size_limits=SIZE_LIMITS, structure=STRUCTURE, remove_size_field=False,
fill_value=None, size_field=None, debug=debug)
gf = noise.significant_detection(
radar, gatefilter=gf, remove_small_features=False, size_bins=SIZE_BINS,
size_limits=SIZE_LIMITS, fill_holes=FILL_HOLES, dilate=DILATE,
structure=STRUCTURE, iterations=1, rays_wrap_around=False,
min_ncp=None, detect_field=None, debug=debug, verbose=verbose)
# Step 2: Doppler velocity correction
if DEALIAS == 'phase':
vdop_corr = dealias_unwrap_phase(
radar, gatefilter=gf, unwrap_unit='sweep', nyquist_vel=None,
rays_wrap_around=True, keep_original=False, vel_field=None,
corr_vel_field=VDOP_CORR_FIELD)
elif DEALIAS == 'region':
vdop_corr = dealias_region_based(
radar, gatefilter=gf, interval_splits=INTERVAL_SPLITS,
interval_limits=None, skip_between_rays=2, skip_along_ray=2,
centered=True, nyquist_vel=None, rays_wrap_around=True,
keep_original=False, vel_field=None,
corr_vel_field=VDOP_CORR_FIELD)
else:
raise ValueError('Unsupported velocity correction routine')
radar.add_field(VDOP_CORR_FIELD, vdop_corr, replace_existing=True)
# Step 3: Reflectivity correction
# Currently no correction procedures are applied to the reflectivity field
# due to minimal attenuation at S-band
refl_corr = radar.fields[REFL_FIELD].copy()
radar.add_field(REFL_CORR_FIELD, refl_corr, replace_existing=True)
# Step 4: Interpolate missing gates
basic_fixes.interpolate_missing(
radar, fields=FILL_FIELDS, interp_window=FILL_WINDOW,
interp_sample=FILL_SAMPLE, kind='mean', rays_wrap_around=False,
fill_value=None, debug=debug, verbose=verbose)
# Add metadata
_add_metadata(radar, filename)
# ARM file name protocols
date_stamp = datetimes_from_radar(radar).min().strftime('%Y%m%d.%H%M%S')
fname = 'nexradwsr88d{}cmac{}.{}.{}.cdf'.format(QF, FN, DL, date_stamp)
# Write CMAC NetCDF file
write_cfradial(os.path.join(outdir, fname), radar, format=FORMAT,
arm_time_variables=True)
return
def read_casa_netcdf(filename, **kwargs):
"""
Read a CASA NetCDF file.
Parameters
----------
filename : str
Name of CASA NetCDF file to read data from.
Returns
-------
radar : Radar
Radar object.
"""
# test for non empty kwargs
_test_arguments(kwargs)
# create metadata retrieval object
filemetadata = FileMetadata('casa_netcdf')
# Open netCDF4 file
dset = netCDF4.Dataset(filename)
nrays = len(dset.dimensions['Radial'])
nbins = len(dset.dimensions['Gate'])
# latitude, longitude and altitude
latitude = filemetadata('latitude')
longitude = filemetadata('longitude')
altitude = filemetadata('altitude')
latitude['data'] = np.array([dset.Latitude], 'float64')
longitude['data'] = np.array([dset.Longitude], 'float64')
altitude['data'] = np.array([dset.Height], 'float64')
# metadata
metadata = filemetadata('metadata')
metadata_mapping = {
'vcp-value': 'vcp', #not in casa
'radarName-value': 'RadarName',
'ConversionPlugin': 'conversion_software', #not in casa
}
for netcdf_attr, metadata_key in metadata_mapping.items():
if netcdf_attr in dset.ncattrs():
metadata[metadata_key] = dset.getncattr(netcdf_attr)
# sweep_start_ray_index, sweep_end_ray_index
sweep_start_ray_index = filemetadata('sweep_start_ray_index')
sweep_end_ray_index = filemetadata('sweep_end_ray_index')
sweep_start_ray_index['data'] = np.array([0], dtype='int32')
sweep_end_ray_index['data'] = np.array([nrays - 1], dtype='int32')
# sweep number
sweep_number = filemetadata('sweep_number')
sweep_number['data'] = np.array([0], dtype='int32')
# sweep_type
scan_type = 'ppi'
# sweep_mode, fixed_angle
sweep_mode = filemetadata('sweep_mode')
fixed_angle = filemetadata('fixed_angle')
sweep_mode['data'] = np.array(1 * ['azimuth_surveillance'])
elev_mean = np.mean(dset.variables['Elevation'])
fixed_angle['data'] = np.array([elev_mean], dtype='float32')
# time
time = filemetadata('time')
start_time = datetime.datetime.utcfromtimestamp(dset.variables['Time'][0])
time['units'] = make_time_unit_str(start_time)
time['data'] = np.zeros((nrays, ), dtype='float64')
# range
_range = filemetadata('range')
step = dset.variables['GateWidth'][0] / 1e3
# * len(dset.variables['GateWidth'][:]))/1e6
_range['data'] = (np.arange(nbins, dtype='float32') * step + step / 2)
_range['meters_to_center_of_first_gate'] = step / 2.
_range['meters_between_gates'] = step
# elevation
elevation = filemetadata('elevation')
elevation_angle = elev_mean
elevation['data'] = np.ones((nrays, ), dtype='float32') * elevation_angle
# azimuth
azimuth = filemetadata('azimuth')
azimuth['data'] = dset.variables['Azimuth'][:]
# fields
# Ryan Note: What is "TypeName"?
# field_name = dset.TypeName
# field_data = np.ma.array(dset.variables[field_name][:])
# if 'MissingData' in dset.ncattrs():
# field_data[field_data == dset.MissingData] = np.ma.masked
# if 'RangeFolded' in dset.ncattrs():
# field_data[field_data == dset.RangeFolded] = np.ma.masked
# fields = {field_name: filemetadata(field_name)}
# fields[field_name]['data'] = field_data
# fields[field_name]['units'] = dset.variables[field_name].Units
# fields[field_name]['_FillValue'] = get_fillvalue()
# borrowing from d3r_gcpex_nc.py
ncvars = dset.variables
keys = [k for k, v in ncvars.items() if v.dimensions == ('Radial', 'Gate')]
fields = {}
for key in keys:
field_name = filemetadata.get_field_name(key)
if field_name is None:
field_name = key
fields[field_name] = _ncvar_to_dict(ncvars[key])
# instrument_parameters
instrument_parameters = {}
if 'PRF-value' in dset.ncattrs():
dic = filemetadata('prt')
prt = 1. / float(dset.getncattr('PRF-value'))
dic['data'] = np.ones((nrays, ), dtype='float32') * prt
instrument_parameters['prt'] = dic
if 'PulseWidth-value' in dset.ncattrs():
dic = filemetadata('pulse_width')
pulse_width = dset.getncattr('PulseWidth-value') * 1.e-6
dic['data'] = np.ones((nrays, ), dtype='float32') * pulse_width
instrument_parameters['pulse_width'] = dic
if 'NyquistVelocity-value' in dset.ncattrs():
dic = filemetadata('nyquist_velocity')
nyquist_velocity = float(dset.getncattr('NyquistVelocity-value'))
dic['data'] = np.ones((nrays, ), dtype='float32') * nyquist_velocity
instrument_parameters['nyquist_velocity'] = dic
if 'Beamwidth' in dset.variables:
dic = filemetadata('radar_beam_width_h')
dic['data'] = dset.variables['Beamwidth'][:]
instrument_parameters['radar_beam_width_h'] = dic
dset.close()
return Radar(time,
_range,
fields,
metadata,
scan_type,
latitude,
longitude,
altitude,
sweep_number,
sweep_mode,
fixed_angle,
sweep_start_ray_index,
sweep_end_ray_index,
azimuth,
elevation,
instrument_parameters=instrument_parameters)