def test_fields_and_refl_heights_to_dict_mrms(self):
"""Ensures correct output from fields_and_refl_heights_to_dict.
In this case, data source is MRMS.
"""
this_dictionary = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=FIELD_NAMES,
data_source=radar_utils.MRMS_SOURCE_ID,
refl_heights_m_asl=REFLECTIVITY_HEIGHTS_M_ASL))
self.assertTrue(this_dictionary == FIELD_TO_HEIGHTS_DICT_MRMS_M_ASL)
def unzip_1day_tar_file(
tar_file_name, field_names, spc_date_string, top_target_directory_name,
refl_heights_m_asl=None):
"""Unzips 1-day tar file (containing raw MYRORSS data for one SPC date).
:param tar_file_name: Path to input file.
:param field_names: 1-D list with names of radar fields.
:param spc_date_string: SPC date (format "yyyymmdd").
:param top_target_directory_name: Name of top-level directory for unzipped
MYRORSS files. This method will create a subdirectory therein for the
SPC date.
:param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres
above sea level).
:return: target_directory_name: Path to output directory.
"""
# Verification.
_ = time_conversion.spc_date_string_to_unix_sec(spc_date_string)
error_checking.assert_is_string_list(field_names)
error_checking.assert_is_numpy_array(
numpy.asarray(field_names), num_dimensions=1)
error_checking.assert_is_string(top_target_directory_name)
# Put azimuthal-shear fields (which are allowed to be missing) at the end.
# This way, if the tar command errors out due to missing data, it will do so
# after unzipping all the non-missing data.
field_names_removed = []
for this_field_name in AZIMUTHAL_RADAR_FIELD_NAMES:
if this_field_name in field_names:
field_names.remove(this_field_name)
field_names_removed.append(this_field_name)
for this_field_name in field_names_removed:
field_names.append(this_field_name)
field_to_heights_dict_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=field_names, data_source=radar_utils.MYRORSS_SOURCE_ID,
refl_heights_m_asl=refl_heights_m_asl))
target_directory_name = '{0:s}/{1:s}/{2:s}'.format(
top_target_directory_name, spc_date_string[:4], spc_date_string
)
field_names = list(field_to_heights_dict_m_asl.keys())
directory_names_to_unzip = []
for this_field_name in field_names:
these_heights_m_asl = field_to_heights_dict_m_asl[this_field_name]
for this_height_m_asl in these_heights_m_asl:
directory_names_to_unzip.append(
myrorss_and_mrms_io.get_relative_dir_for_raw_files(
field_name=this_field_name,
data_source=radar_utils.MYRORSS_SOURCE_ID,
height_m_asl=this_height_m_asl))
unzipping.unzip_tar(
tar_file_name,
target_directory_name=target_directory_name,
file_and_dir_names_to_unzip=directory_names_to_unzip)
return target_directory_name
def remove_unzipped_data_1day(
spc_date_string, top_directory_name,
field_names=DEFAULT_FIELDS_TO_REMOVE,
refl_heights_m_asl=DEFAULT_REFL_HEIGHTS_TO_REMOVE_M_ASL):
"""Removes unzipped MYRORSS data for one SPC date.
Basically, this method cleans up after unzip_1day_tar_file.
:param spc_date_string: SPC date (format "yyyymmdd").
:param top_directory_name: Name of top-level directory with unzipped MYRORSS
files. This method will find the subdirectory in `top_directory_name`
for the given SPC date.
:param field_names: 1-D list with names of radar fields. Only these will be
deleted.
:param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres
above sea level).
"""
spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
spc_date_string)
field_to_heights_dict_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=field_names, data_source=radar_utils.MYRORSS_SOURCE_ID,
refl_heights_m_asl=refl_heights_m_asl))
for this_field_name in list(field_to_heights_dict_m_asl.keys()):
these_heights_m_asl = field_to_heights_dict_m_asl[this_field_name]
for this_height_m_asl in these_heights_m_asl:
example_file_name = myrorss_and_mrms_io.find_raw_file(
unix_time_sec=spc_date_unix_sec,
spc_date_string=spc_date_string, field_name=this_field_name,
data_source=radar_utils.MYRORSS_SOURCE_ID,
top_directory_name=top_directory_name,
height_m_asl=this_height_m_asl,
raise_error_if_missing=False)
example_directory_name, _ = os.path.split(example_file_name)
directory_name_parts = example_directory_name.split('/')
remove_all_heights = False
if this_field_name == radar_utils.REFL_NAME:
if (set(these_heights_m_asl) ==
set(DEFAULT_REFL_HEIGHTS_TO_REMOVE_M_ASL)):
remove_all_heights = True
dir_name_to_remove = '/'.join(directory_name_parts[:-1])
else:
dir_name_to_remove = '/'.join(directory_name_parts)
else:
dir_name_to_remove = '/'.join(directory_name_parts[:-1])
if os.path.isdir(dir_name_to_remove):
print('Removing directory "{0:s}"...'.format(
dir_name_to_remove))
shutil.rmtree(dir_name_to_remove, ignore_errors=True)
if remove_all_heights:
break
def write_field_to_myrorss_file(field_matrix,
netcdf_file_name,
field_name,
metadata_dict,
height_m_asl=None):
"""Writes field to MYRORSS-formatted file.
M = number of rows (unique grid-point latitudes)
N = number of columns (unique grid-point longitudes)
:param field_matrix: M-by-N numpy array with one radar variable at one time.
Latitude should increase down each column, and longitude should increase
to the right along each row.
:param netcdf_file_name: Path to output file.
:param field_name: Name of radar field in GewitterGefahr format.
:param metadata_dict: Dictionary created by either
`gridrad_io.read_metadata_from_full_grid_file` or
`read_metadata_from_raw_file`.
:param height_m_asl: Height of radar field (metres above sea level).
"""
if field_name == radar_utils.REFL_NAME:
field_to_heights_dict_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=[field_name],
data_source=radar_utils.MYRORSS_SOURCE_ID,
refl_heights_m_asl=numpy.array([height_m_asl])))
else:
field_to_heights_dict_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=[field_name],
data_source=radar_utils.MYRORSS_SOURCE_ID))
field_name = list(field_to_heights_dict_m_asl.keys())[0]
radar_height_m_asl = field_to_heights_dict_m_asl[field_name][0]
if field_name in radar_utils.ECHO_TOP_NAMES:
field_matrix = METRES_TO_KM * field_matrix
field_name_myrorss = radar_utils.field_name_new_to_orig(
field_name=field_name, data_source_name=radar_utils.MYRORSS_SOURCE_ID)
file_system_utils.mkdir_recursive_if_necessary(file_name=netcdf_file_name)
netcdf_dataset = Dataset(netcdf_file_name,
'w',
format='NETCDF3_64BIT_OFFSET')
netcdf_dataset.setncattr(FIELD_NAME_COLUMN_ORIG, field_name_myrorss)
netcdf_dataset.setncattr('DataType', 'SparseLatLonGrid')
netcdf_dataset.setncattr(
NW_GRID_POINT_LAT_COLUMN_ORIG,
rounder.round_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(
NW_GRID_POINT_LNG_COLUMN_ORIG,
rounder.round_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LNG_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(HEIGHT_COLUMN_ORIG,
METRES_TO_KM * numpy.float(radar_height_m_asl))
netcdf_dataset.setncattr(
UNIX_TIME_COLUMN_ORIG,
numpy.int32(metadata_dict[radar_utils.UNIX_TIME_COLUMN]))
netcdf_dataset.setncattr('FractionalTime', 0.)
netcdf_dataset.setncattr('attributes', ' ColorMap SubType Unit')
netcdf_dataset.setncattr('ColorMap-unit', 'dimensionless')
netcdf_dataset.setncattr('ColorMap-value', '')
netcdf_dataset.setncattr('SubType-unit', 'dimensionless')
netcdf_dataset.setncattr('SubType-value', numpy.float(radar_height_m_asl))
netcdf_dataset.setncattr('Unit-unit', 'dimensionless')
netcdf_dataset.setncattr('Unit-value', 'dimensionless')
netcdf_dataset.setncattr(
LAT_SPACING_COLUMN_ORIG,
rounder.round_to_nearest(metadata_dict[radar_utils.LAT_SPACING_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(
LNG_SPACING_COLUMN_ORIG,
rounder.round_to_nearest(metadata_dict[radar_utils.LNG_SPACING_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(SENTINEL_VALUE_COLUMNS_ORIG[0],
numpy.double(-99000.))
netcdf_dataset.setncattr(SENTINEL_VALUE_COLUMNS_ORIG[1],
numpy.double(-99001.))
min_latitude_deg = metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN] - (
metadata_dict[radar_utils.LAT_SPACING_COLUMN] *
(metadata_dict[radar_utils.NUM_LAT_COLUMN] - 1))
unique_grid_point_lats_deg, unique_grid_point_lngs_deg = (
grids.get_latlng_grid_points(
min_latitude_deg=min_latitude_deg,
min_longitude_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN],
num_rows=metadata_dict[radar_utils.NUM_LAT_COLUMN],
num_columns=metadata_dict[radar_utils.NUM_LNG_COLUMN]))
num_grid_rows = len(unique_grid_point_lats_deg)
num_grid_columns = len(unique_grid_point_lngs_deg)
field_vector = numpy.reshape(field_matrix,
num_grid_rows * num_grid_columns)
grid_point_lat_matrix, grid_point_lng_matrix = (
grids.latlng_vectors_to_matrices(unique_grid_point_lats_deg,
unique_grid_point_lngs_deg))
grid_point_lat_vector = numpy.reshape(grid_point_lat_matrix,
num_grid_rows * num_grid_columns)
grid_point_lng_vector = numpy.reshape(grid_point_lng_matrix,
num_grid_rows * num_grid_columns)
real_value_indices = numpy.where(numpy.invert(
numpy.isnan(field_vector)))[0]
netcdf_dataset.createDimension(NUM_LAT_COLUMN_ORIG, num_grid_rows - 1)
netcdf_dataset.createDimension(NUM_LNG_COLUMN_ORIG, num_grid_columns - 1)
netcdf_dataset.createDimension(NUM_PIXELS_COLUMN_ORIG,
len(real_value_indices))
row_index_vector, column_index_vector = radar_utils.latlng_to_rowcol(
grid_point_lat_vector,
grid_point_lng_vector,
nw_grid_point_lat_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN])
netcdf_dataset.createVariable(field_name_myrorss, numpy.single,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.createVariable(GRID_ROW_COLUMN_ORIG, numpy.int16,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.createVariable(GRID_COLUMN_COLUMN_ORIG, numpy.int16,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.createVariable(NUM_GRID_CELL_COLUMN_ORIG, numpy.int32,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.variables[field_name_myrorss].setncattr(
'BackgroundValue', numpy.int32(-99900))
netcdf_dataset.variables[field_name_myrorss].setncattr(
'units', 'dimensionless')
netcdf_dataset.variables[field_name_myrorss].setncattr(
'NumValidRuns', numpy.int32(len(real_value_indices)))
netcdf_dataset.variables[field_name_myrorss][:] = field_vector[
real_value_indices]
netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG][:] = (
row_index_vector[real_value_indices])
netcdf_dataset.variables[GRID_COLUMN_COLUMN_ORIG][:] = (
column_index_vector[real_value_indices])
netcdf_dataset.variables[NUM_GRID_CELL_COLUMN_ORIG][:] = (numpy.full(
len(real_value_indices), 1, dtype=int))
netcdf_dataset.close()