def test_field_name_new_to_orig_gridrad_invalid(self):
"""Ensures correct output from field_name_new_to_orig.
In this case, trying to convert field name to GridRad format, but field
does not exist in GridRad.
"""
with self.assertRaises(ValueError):
radar_utils.field_name_new_to_orig(
LL_SHEAR_NAME_NEW, data_source=radar_utils.GRIDRAD_SOURCE_ID)
def test_field_name_new_to_orig_mrms_invalid(self):
"""Ensures correct output from field_name_new_to_orig.
In this case, trying to convert field name to MRMS format, but field
does not exist in MRMS.
"""
with self.assertRaises(ValueError):
radar_utils.field_name_new_to_orig(
radar_utils.DIFFERENTIAL_REFL_NAME,
data_source=radar_utils.MRMS_SOURCE_ID)
def read_field_from_full_grid_file(netcdf_file_name,
field_name=None,
metadata_dict=None,
raise_error_if_fails=True):
"""Reads one radar field from full-grid (not sparse-grid) file.
This file should contain all radar variables for one time step.
M = number of rows (unique grid-point latitudes)
N = number of columns (unique grid-point longitudes)
H = number of height levels (unique grid-point heights)
:param netcdf_file_name: Path to input file.
:param field_name: Name of radar field.
:param metadata_dict: Dictionary created by
read_metadata_from_full_grid_file.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error. If False and file cannot be
opened, will return None for all output variables.
:return: field_matrix: H-by-M-by-N numpy array with values of radar field.
:return: grid_point_heights_m_asl: length-H numpy array of height levels
(integer metres above sea level). If array is increasing (decreasing),
height increases (decreases) with the first index of field_matrix.
:return: grid_point_latitudes_deg: length-M numpy array of grid-point
latitudes (deg N). If array is increasing (decreasing), latitude
increases (decreases) with the second index of field_matrix.
:return: grid_point_longitudes_deg: length-N numpy array of grid-point
longitudes (deg N). If array is increasing (decreasing), latitude
increases (decreases) with the third index of field_matrix.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None, None, None, None
field_name_orig = radar_utils.field_name_new_to_orig(
field_name=field_name, data_source_name=radar_utils.GRIDRAD_SOURCE_ID)
field_matrix = numpy.array(
netcdf_dataset.variables[field_name_orig][0, :, :, :])
grid_point_latitudes_deg = numpy.array(
netcdf_dataset.variables[LATITUDE_NAME_ORIG])
grid_point_longitudes_deg = lng_conversion.convert_lng_positive_in_west(
numpy.array(netcdf_dataset.variables[LONGITUDE_NAME_ORIG]))
_check_grid_points(grid_point_latitudes_deg=grid_point_latitudes_deg,
grid_point_longitudes_deg=grid_point_longitudes_deg,
metadata_dict=metadata_dict)
grid_point_heights_m_asl = KM_TO_METRES * numpy.array(
netcdf_dataset.variables[HEIGHT_NAME_ORIG])
grid_point_heights_m_asl = numpy.round(grid_point_heights_m_asl).astype(
int)
netcdf_dataset.close()
return (field_matrix, grid_point_heights_m_asl, grid_point_latitudes_deg,
grid_point_longitudes_deg)
def test_field_name_new_to_orig_mrms_valid(self):
"""Ensures correct output from field_name_new_to_orig.
In this case, field name is being converted to MRMS format.
"""
this_field_name_mrms = radar_utils.field_name_new_to_orig(
LL_SHEAR_NAME_NEW, data_source=radar_utils.MRMS_SOURCE_ID)
self.assertTrue(this_field_name_mrms == LL_SHEAR_NAME_MRMS)
def test_field_name_new_to_orig_myrorss_valid(self):
"""Ensures correct output from field_name_new_to_orig.
In this case, field name is being converted to MYRORSS format.
"""
this_field_name_myrorss = radar_utils.field_name_new_to_orig(
field_name=LL_SHEAR_NAME,
data_source_name=radar_utils.MYRORSS_SOURCE_ID)
self.assertTrue(this_field_name_myrorss == LL_SHEAR_NAME_MYRORSS)
def test_field_name_new_to_orig_gridrad_valid(self):
"""Ensures correct output from field_name_new_to_orig.
In this case, field name is being converted to GridRad format.
"""
this_field_name_gridrad = radar_utils.field_name_new_to_orig(
radar_utils.DIFFERENTIAL_REFL_NAME,
data_source=radar_utils.GRIDRAD_SOURCE_ID)
self.assertTrue(this_field_name_gridrad ==
radar_utils.DIFFERENTIAL_REFL_NAME_GRIDRAD)
def fields_and_refl_heights_to_pairs(field_names, heights_m_asl):
"""Converts unique arrays (field names and heights) to non-unique ones.
F = number of fields
H = number of heights
N = F * H = number of field/height pairs
:param field_names: length-F list with names of radar fields in
GewitterGefahr format.
:param heights_m_asl: length-H numpy array of heights (metres above sea
level).
:return: field_name_by_pair: length-N list of field names.
:return: height_by_pair_m_asl: length-N numpy array of corresponding heights
(metres above sea level).
"""
error_checking.assert_is_string_list(field_names)
error_checking.assert_is_numpy_array(numpy.array(field_names),
num_dimensions=1)
radar_utils.check_heights(data_source=radar_utils.GRIDRAD_SOURCE_ID,
heights_m_asl=heights_m_asl)
field_name_by_pair = []
height_by_pair_m_asl = numpy.array([], dtype=int)
for this_field_name in field_names:
radar_utils.field_name_new_to_orig(
field_name=this_field_name,
data_source_name=radar_utils.GRIDRAD_SOURCE_ID)
field_name_by_pair += [this_field_name] * len(heights_m_asl)
height_by_pair_m_asl = numpy.concatenate(
(height_by_pair_m_asl, heights_m_asl))
return field_name_by_pair, height_by_pair_m_asl
def get_relative_dir_for_raw_files(field_name, data_source, height_m_asl=None):
"""Generates relative path for raw files.
:param field_name: Name of radar field in GewitterGefahr format.
:param data_source: Data source (string).
:param height_m_asl: Radar height (metres above sea level).
:return: relative_directory_name: Relative path for raw files.
"""
if field_name == radar_utils.REFL_NAME:
radar_utils.check_heights(data_source=data_source,
heights_m_asl=numpy.array([height_m_asl]),
field_name=radar_utils.REFL_NAME)
else:
height_m_asl = radar_utils.get_valid_heights(data_source=data_source,
field_name=field_name)[0]
return '{0:s}/{1:05.2f}'.format(
radar_utils.field_name_new_to_orig(field_name=field_name,
data_source_name=data_source),
float(height_m_asl) * METRES_TO_KM)
def write_field_to_myrorss_file(field_matrix,
netcdf_file_name,
field_name,
metadata_dict,
height_m_asl=None):
"""Writes field to MYRORSS-formatted file.
M = number of rows (unique grid-point latitudes)
N = number of columns (unique grid-point longitudes)
:param field_matrix: M-by-N numpy array with one radar variable at one time.
Latitude should increase down each column, and longitude should increase
to the right along each row.
:param netcdf_file_name: Path to output file.
:param field_name: Name of radar field in GewitterGefahr format.
:param metadata_dict: Dictionary created by either
`gridrad_io.read_metadata_from_full_grid_file` or
`read_metadata_from_raw_file`.
:param height_m_asl: Height of radar field (metres above sea level).
"""
if field_name == radar_utils.REFL_NAME:
field_to_heights_dict_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=[field_name],
data_source=radar_utils.MYRORSS_SOURCE_ID,
refl_heights_m_asl=numpy.array([height_m_asl])))
else:
field_to_heights_dict_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=[field_name],
data_source=radar_utils.MYRORSS_SOURCE_ID))
field_name = list(field_to_heights_dict_m_asl.keys())[0]
radar_height_m_asl = field_to_heights_dict_m_asl[field_name][0]
if field_name in radar_utils.ECHO_TOP_NAMES:
field_matrix = METRES_TO_KM * field_matrix
field_name_myrorss = radar_utils.field_name_new_to_orig(
field_name=field_name, data_source_name=radar_utils.MYRORSS_SOURCE_ID)
file_system_utils.mkdir_recursive_if_necessary(file_name=netcdf_file_name)
netcdf_dataset = Dataset(netcdf_file_name,
'w',
format='NETCDF3_64BIT_OFFSET')
netcdf_dataset.setncattr(FIELD_NAME_COLUMN_ORIG, field_name_myrorss)
netcdf_dataset.setncattr('DataType', 'SparseLatLonGrid')
netcdf_dataset.setncattr(
NW_GRID_POINT_LAT_COLUMN_ORIG,
rounder.round_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(
NW_GRID_POINT_LNG_COLUMN_ORIG,
rounder.round_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LNG_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(HEIGHT_COLUMN_ORIG,
METRES_TO_KM * numpy.float(radar_height_m_asl))
netcdf_dataset.setncattr(
UNIX_TIME_COLUMN_ORIG,
numpy.int32(metadata_dict[radar_utils.UNIX_TIME_COLUMN]))
netcdf_dataset.setncattr('FractionalTime', 0.)
netcdf_dataset.setncattr('attributes', ' ColorMap SubType Unit')
netcdf_dataset.setncattr('ColorMap-unit', 'dimensionless')
netcdf_dataset.setncattr('ColorMap-value', '')
netcdf_dataset.setncattr('SubType-unit', 'dimensionless')
netcdf_dataset.setncattr('SubType-value', numpy.float(radar_height_m_asl))
netcdf_dataset.setncattr('Unit-unit', 'dimensionless')
netcdf_dataset.setncattr('Unit-value', 'dimensionless')
netcdf_dataset.setncattr(
LAT_SPACING_COLUMN_ORIG,
rounder.round_to_nearest(metadata_dict[radar_utils.LAT_SPACING_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(
LNG_SPACING_COLUMN_ORIG,
rounder.round_to_nearest(metadata_dict[radar_utils.LNG_SPACING_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(SENTINEL_VALUE_COLUMNS_ORIG[0],
numpy.double(-99000.))
netcdf_dataset.setncattr(SENTINEL_VALUE_COLUMNS_ORIG[1],
numpy.double(-99001.))
min_latitude_deg = metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN] - (
metadata_dict[radar_utils.LAT_SPACING_COLUMN] *
(metadata_dict[radar_utils.NUM_LAT_COLUMN] - 1))
unique_grid_point_lats_deg, unique_grid_point_lngs_deg = (
grids.get_latlng_grid_points(
min_latitude_deg=min_latitude_deg,
min_longitude_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN],
num_rows=metadata_dict[radar_utils.NUM_LAT_COLUMN],
num_columns=metadata_dict[radar_utils.NUM_LNG_COLUMN]))
num_grid_rows = len(unique_grid_point_lats_deg)
num_grid_columns = len(unique_grid_point_lngs_deg)
field_vector = numpy.reshape(field_matrix,
num_grid_rows * num_grid_columns)
grid_point_lat_matrix, grid_point_lng_matrix = (
grids.latlng_vectors_to_matrices(unique_grid_point_lats_deg,
unique_grid_point_lngs_deg))
grid_point_lat_vector = numpy.reshape(grid_point_lat_matrix,
num_grid_rows * num_grid_columns)
grid_point_lng_vector = numpy.reshape(grid_point_lng_matrix,
num_grid_rows * num_grid_columns)
real_value_indices = numpy.where(numpy.invert(
numpy.isnan(field_vector)))[0]
netcdf_dataset.createDimension(NUM_LAT_COLUMN_ORIG, num_grid_rows - 1)
netcdf_dataset.createDimension(NUM_LNG_COLUMN_ORIG, num_grid_columns - 1)
netcdf_dataset.createDimension(NUM_PIXELS_COLUMN_ORIG,
len(real_value_indices))
row_index_vector, column_index_vector = radar_utils.latlng_to_rowcol(
grid_point_lat_vector,
grid_point_lng_vector,
nw_grid_point_lat_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN])
netcdf_dataset.createVariable(field_name_myrorss, numpy.single,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.createVariable(GRID_ROW_COLUMN_ORIG, numpy.int16,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.createVariable(GRID_COLUMN_COLUMN_ORIG, numpy.int16,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.createVariable(NUM_GRID_CELL_COLUMN_ORIG, numpy.int32,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.variables[field_name_myrorss].setncattr(
'BackgroundValue', numpy.int32(-99900))
netcdf_dataset.variables[field_name_myrorss].setncattr(
'units', 'dimensionless')
netcdf_dataset.variables[field_name_myrorss].setncattr(
'NumValidRuns', numpy.int32(len(real_value_indices)))
netcdf_dataset.variables[field_name_myrorss][:] = field_vector[
real_value_indices]
netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG][:] = (
row_index_vector[real_value_indices])
netcdf_dataset.variables[GRID_COLUMN_COLUMN_ORIG][:] = (
column_index_vector[real_value_indices])
netcdf_dataset.variables[NUM_GRID_CELL_COLUMN_ORIG][:] = (numpy.full(
len(real_value_indices), 1, dtype=int))
netcdf_dataset.close()
"""Unit tests for myrorss_and_mrms_io.py."""
import unittest
import numpy
import pandas
from gewittergefahr.gg_io import myrorss_and_mrms_io
from gewittergefahr.gg_utils import radar_utils
TOLERANCE = 1e-6
LL_SHEAR_NAME = radar_utils.LOW_LEVEL_SHEAR_NAME
LL_SHEAR_NAME_MYRORSS = radar_utils.field_name_new_to_orig(
field_name=radar_utils.LOW_LEVEL_SHEAR_NAME,
data_source_name=radar_utils.MYRORSS_SOURCE_ID)
LL_SHEAR_NAME_MRMS = radar_utils.field_name_new_to_orig(
field_name=radar_utils.LOW_LEVEL_SHEAR_NAME,
data_source_name=radar_utils.MRMS_SOURCE_ID)
# The following constants are used to test _get_pathless_raw_file_pattern and
# _get_pathless_raw_file_name.
FILE_TIME_UNIX_SEC = 1507234802
FILE_SPC_DATE_STRING = '20171005'
PATHLESS_ZIPPED_FILE_NAME = '20171005-202002.netcdf.gz'
PATHLESS_UNZIPPED_FILE_NAME = '20171005-202002.netcdf'
PATHLESS_FILE_PATTERN = '20171005-2020*.netcdf*'
# The following constants are used to test _remove_sentinels_from_sparse_grid.
THESE_GRID_ROWS = numpy.linspace(0, 10, num=11, dtype=int)
THESE_GRID_COLUMNS = numpy.linspace(0, 10, num=11, dtype=int)
THESE_NUM_GRID_CELLS = numpy.linspace(0, 10, num=11, dtype=int)