def test_missing_variable(bad_mwr_file):
with pytest.raises(RuntimeError):
Mwr(bad_mwr_file)
def test_init_lwp_error(fake_mwr_file):
obj = Mwr(fake_mwr_file)
lwp = obj.data['lwp'][:]
lwp_error = obj.data['lwp_error'][:]
assert_array_equal(lwp_error, np.sqrt((0.25*lwp)**2 + 50**2))
def test_rebin_to_grid(fake_mwr_file):
obj = Mwr(fake_mwr_file)
time_new = np.array([1.6, 2.6])
obj.rebin_to_grid(time_new)
result = np.array([2.5, 0.2])
assert_array_equal(obj.data['lwp'][:], result)
def test_init_lwp_data(fake_mwr_file):
obj = Mwr(fake_mwr_file)
result = np.array([0.1, 2.5, 0.0, 0.2, 0.0])
assert_array_equal(obj.data['lwp'][:], result)
def generate_categorize(input_files: dict,
output_file: str,
uuid: Optional[str] = None) -> str:
"""Generates Cloudnet Level 1c categorize file.
The measurements are rebinned into a common height / time grid,
and classified as different types of scatterers such as ice, liquid,
insects, etc. Next, the radar signal is corrected for atmospheric
attenuation, and error estimates are computed. Results are saved
in *ouput_file* which is a compressed netCDF4 file.
Args:
input_files: dict containing file names for calibrated `radar`, `lidar`, `model` and
`mwr` files.
output_file: Full path of the output file.
uuid: Set specific UUID for the file.
Returns:
UUID of the generated file.
Raises:
RuntimeError: Failed to create the categorize file.
Notes:
Separate mwr-file is not needed when using RPG cloud radar which
measures liquid water path. Then, the radar file can be used as
a mwr-file as well, i.e. {'mwr': 'radar.nc'}.
Examples:
>>> from cloudnetpy.categorize import generate_categorize
>>> input_files = {'radar': 'radar.nc',
'lidar': 'lidar.nc',
'model': 'model.nc',
'mwr': 'mwr.nc'}
>>> generate_categorize(input_files, 'output.nc')
"""
def _interpolate_to_cloudnet_grid() -> list:
wl_band = utils.get_wl_band(data["radar"].radar_frequency)
data["model"].interpolate_to_common_height(wl_band)
data["model"].interpolate_to_grid(time, height)
data["mwr"].rebin_to_grid(time)
radar_data_gap_indices = data["radar"].rebin_to_grid(time)
lidar_data_gap_indices = data["lidar"].interpolate_to_grid(
time, height)
bad_time_indices = list(
set(radar_data_gap_indices + lidar_data_gap_indices))
valid_ind = [
ind for ind in range(len(time)) if ind not in bad_time_indices
]
return valid_ind
def _screen_bad_time_indices(valid_indices: list) -> None:
n_time_full = len(time)
data["radar"].time = time[valid_indices]
for var in ("radar", "lidar", "mwr", "model"):
for key, item in data[var].data.items():
if utils.isscalar(item.data):
continue
array = item[:]
if array.shape[0] == n_time_full:
if array.ndim == 1:
array = array[valid_indices]
elif array.ndim == 2:
array = array[valid_indices, :]
else:
continue
data[var].data[key].data = array
for key, item in data["model"].data_dense.items():
data["model"].data_dense[key] = item[valid_indices, :]
def _prepare_output() -> dict:
data["radar"].add_meta()
data["model"].screen_sparse_fields()
for key in ("category_bits", "rain_rate"):
data["radar"].append_data(getattr(classification, key), key)
for key in ("radar_liquid_atten", "radar_gas_atten"):
data["radar"].append_data(attenuations[key], key)
data["radar"].append_data(quality["quality_bits"], "quality_bits")
return {
**data["radar"].data,
**data["lidar"].data,
**data["model"].data,
**data["model"].data_sparse,
**data["mwr"].data,
}
def _define_dense_grid():
return utils.time_grid(), data["radar"].height
def _close_all():
for obj in data.values():
obj.close()
data = {
"radar": Radar(input_files["radar"]),
"lidar": Lidar(input_files["lidar"]),
"mwr": Mwr(input_files["mwr"]),
}
assert data["radar"].altitude is not None
data["model"] = Model(input_files["model"], data["radar"].altitude)
time, height = _define_dense_grid()
valid_ind = _interpolate_to_cloudnet_grid()
_screen_bad_time_indices(valid_ind)
if "rpg" in data["radar"].type.lower(
) or "basta" in data["radar"].type.lower():
data["radar"].filter_speckle_noise()
data["radar"].filter_1st_gate_artifact()
for variable in ("v", "v_sigma", "ldr"):
data["radar"].filter_stripes(variable)
data["radar"].remove_incomplete_pixels()
data["model"].calc_wet_bulb()
classification = classify.classify_measurements(data)
attenuations = atmos.get_attenuations(data, classification)
data["radar"].correct_atten(attenuations)
data["radar"].calc_errors(attenuations, classification)
quality = classify.fetch_quality(data, classification, attenuations)
cloudnet_arrays = _prepare_output()
date = data["radar"].get_date()
attributes = output.add_time_attribute(CATEGORIZE_ATTRIBUTES, date)
attributes = output.add_time_attribute(attributes, date, "model_time")
attributes = output.add_source_attribute(attributes, data)
output.update_attributes(cloudnet_arrays, attributes)
uuid = _save_cat(output_file, data, cloudnet_arrays, uuid)
_close_all()
return uuid
def generate_categorize(input_files, output_file):
"""Generates Cloudnet categorize file.
The measurements are rebinned into a common height / time grid,
and classified as different types of scatterers such as ice, liquid,
insects, etc. Next, the radar signal is corrected for atmospheric
attenuation, and error estimates are computed. Results are saved
in *ouput_file* which is a compressed netCDF4 file.
Args:
input_files (dict): dict containing file names for calibrated
`radar`, `lidar`, `model` and `mwr` files.
output_file (str): Full path of the output file.
Raises:
RuntimeError: Failed to create the categorize file.
Notes:
Separate mwr-file is not needed when using RPG cloud radar which
measures liquid water path. Then, the radar file can be used as
a mwr-file as well, i.e. {'mwr': 'radar.nc'}.
Examples:
>>> from cloudnetpy.categorize import generate_categorize
>>> input_files = {'radar': 'radar.nc',
'lidar': 'lidar.nc',
'model': 'model.nc',
'mwr': 'mwr.nc'}
>>> generate_categorize(input_files, 'output.nc')
"""
def _interpolate_to_cloudnet_grid():
model.interpolate_to_common_height(radar.wl_band)
model.interpolate_to_grid(time, height)
mwr.rebin_to_grid(time)
radar.rebin_to_grid(time)
lidar.rebin_to_grid(time, height)
def _prepare_output():
radar.add_meta()
model.screen_sparse_fields()
for key in ('category_bits', 'insect_prob', 'is_rain',
'is_undetected_melting'):
radar.append_data(getattr(classification, key), key)
for key in ('radar_liquid_atten', 'radar_gas_atten'):
radar.append_data(attenuations[key], key)
radar.append_data(quality['quality_bits'], 'quality_bits')
return {
**radar.data,
**lidar.data,
**model.data,
**model.data_sparse,
**mwr.data
}
def _define_dense_grid():
return utils.time_grid(), radar.height
def _close_all():
for obj in (radar, lidar, model, mwr):
obj.close()
radar = Radar(input_files['radar'])
lidar = Lidar(input_files['lidar'])
model = Model(input_files['model'], radar.altitude)
mwr = Mwr(input_files['mwr'])
time, height = _define_dense_grid()
_interpolate_to_cloudnet_grid()
if 'rpg' in radar.type.lower():
radar.filter_speckle_noise()
radar.remove_incomplete_pixels()
model.calc_wet_bulb()
classification = classify.classify_measurements(radar, lidar, model, mwr)
attenuations = atmos.get_attenuations(model, mwr, classification)
radar.correct_atten(attenuations)
radar.calc_errors(attenuations, classification)
quality = classify.fetch_quality(radar, lidar, classification,
attenuations)
output_data = _prepare_output()
output.update_attributes(output_data, CATEGORIZE_ATTRIBUTES)
_save_cat(output_file, radar, lidar, model, output_data)
_close_all()
def generate_categorize(input_files: dict,
output_file: str,
keep_uuid: Optional[bool] = False,
uuid: Optional[str] = None) -> str:
"""Generates Cloudnet Level 1c categorize file.
The measurements are rebinned into a common height / time grid,
and classified as different types of scatterers such as ice, liquid,
insects, etc. Next, the radar signal is corrected for atmospheric
attenuation, and error estimates are computed. Results are saved
in *ouput_file* which is a compressed netCDF4 file.
Args:
input_files: dict containing file names for calibrated `radar`, `lidar`, `model` and
`mwr` files.
output_file: Full path of the output file.
keep_uuid: If True, keeps the UUID of the old file, if that exists. Default is False
when new UUID is generated.
uuid: Set specific UUID for the file.
Returns:
UUID of the generated file.
Raises:
RuntimeError: Failed to create the categorize file.
Notes:
Separate mwr-file is not needed when using RPG cloud radar which
measures liquid water path. Then, the radar file can be used as
a mwr-file as well, i.e. {'mwr': 'radar.nc'}.
Examples:
>>> from cloudnetpy.categorize import generate_categorize
>>> input_files = {'radar': 'radar.nc',
'lidar': 'lidar.nc',
'model': 'model.nc',
'mwr': 'mwr.nc'}
>>> generate_categorize(input_files, 'output.nc')
"""
def _interpolate_to_cloudnet_grid():
wl_band = utils.get_wl_band(data['radar'].radar_frequency)
data['model'].interpolate_to_common_height(wl_band)
data['model'].interpolate_to_grid(time, height)
data['mwr'].rebin_to_grid(time)
data['radar'].rebin_to_grid(time)
data['lidar'].rebin_to_grid(time, height)
def _prepare_output() -> dict:
data['radar'].add_meta()
data['model'].screen_sparse_fields()
for key in ('category_bits', 'insect_prob', 'is_rain',
'is_undetected_melting'):
data['radar'].append_data(getattr(classification, key), key)
for key in ('radar_liquid_atten', 'radar_gas_atten'):
data['radar'].append_data(attenuations[key], key)
data['radar'].append_data(quality['quality_bits'], 'quality_bits')
return {
**data['radar'].data,
**data['lidar'].data,
**data['model'].data,
**data['model'].data_sparse,
**data['mwr'].data
}
def _define_dense_grid():
return utils.time_grid(), data['radar'].height
def _close_all():
for obj in data.values():
obj.close()
data = {
'radar': Radar(input_files['radar']),
'lidar': Lidar(input_files['lidar']),
'mwr': Mwr(input_files['mwr'])
}
data['model'] = Model(input_files['model'], data['radar'].altitude)
time, height = _define_dense_grid()
_interpolate_to_cloudnet_grid()
if 'rpg' in data['radar'].type.lower():
data['radar'].filter_speckle_noise()
data['radar'].filter_1st_gate_artifact()
for variable in ('v', 'v_sigma', 'ldr'):
data['radar'].filter_stripes(variable)
data['radar'].remove_incomplete_pixels()
data['model'].calc_wet_bulb()
classification = classify.classify_measurements(data)
attenuations = atmos.get_attenuations(data, classification)
data['radar'].correct_atten(attenuations)
data['radar'].calc_errors(attenuations, classification)
quality = classify.fetch_quality(data, classification, attenuations)
cloudnet_arrays = _prepare_output()
date = data['radar'].get_date()
attributes = output.add_time_attribute(CATEGORIZE_ATTRIBUTES, date)
output.update_attributes(cloudnet_arrays, attributes)
uuid = _save_cat(output_file, data, cloudnet_arrays, keep_uuid, uuid)
_close_all()
return uuid
def test_rebin_to_grid(fake_mwr_file):
obj = Mwr(fake_mwr_file)
time_new = np.array([1.4, 2.8])
obj.rebin_to_grid(time_new)
result = np.array([1.2, 0.2])
assert_array_equal(obj.data["lwp"][:], result)