def test_normalize_soundings_z(self):
"""Ensures correct output from normalize_soundings; z-score method."""
this_sounding_matrix = dl_utils.normalize_soundings(
sounding_matrix=copy.deepcopy(SOUNDING_MATRIX_UNNORMALIZED),
field_names=SOUNDING_FIELD_NAMES,
normalization_type_string=dl_utils.Z_NORMALIZATION_TYPE_STRING,
normalization_param_file_name=None,
test_mode=True,
normalization_table=SOUNDING_NORMALIZATION_TABLE)
self.assertTrue(
numpy.allclose(this_sounding_matrix,
SOUNDING_MATRIX_Z_SCORES,
atol=TOLERANCE))
def init_function(matrix_dimensions):
"""Initializes model input to climatological means.
This function uses one mean for each radar field/height pair and each
sounding field/height pair, rather than one per field altogether, to
create "realistic" vertical profiles.
If len(matrix_dimensions) = 3, this function creates initial soundings.
If len(matrix_dimensions) = 4 and myrorss_2d3d = False, creates initial
2-D radar images with all fields in `training_option_dict`.
If len(matrix_dimensions) = 4 and myrorss_2d3d = True, creates initial
2-D radar images with only azimuthal-shear fields in
`training_option_dict`.
If len(matrix_dimensions) = 5 and myrorss_2d3d = False, creates initial
3-D radar images with all fields in `training_option_dict`.
If len(matrix_dimensions) = 5 and myrorss_2d3d = True, creates initial
3-D reflectivity images.
:param matrix_dimensions: numpy array with desired dimensions
for initialized model input.
:return: initial_matrix: numpy array with the given dimensions.
"""
initial_matrix = numpy.full(matrix_dimensions, numpy.nan)
if len(matrix_dimensions) == 5:
if model_metadata_dict[cnn.USE_2D3D_CONVOLUTION_KEY]:
radar_field_names = [radar_utils.REFL_NAME]
else:
radar_field_names = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
radar_heights_m_agl = training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]
for j in range(len(radar_field_names)):
for k in range(len(radar_heights_m_agl)):
this_key = (radar_field_names[j], radar_heights_m_agl[k])
initial_matrix[..., k, j] = radar_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
return dl_utils.normalize_radar_images(
radar_image_matrix=initial_matrix,
field_names=radar_field_names,
normalization_type_string=training_option_dict[
trainval_io.NORMALIZATION_TYPE_KEY],
normalization_param_file_name=training_option_dict[
trainval_io.NORMALIZATION_FILE_KEY],
test_mode=test_mode,
min_normalized_value=training_option_dict[
trainval_io.MIN_NORMALIZED_VALUE_KEY],
max_normalized_value=training_option_dict[
trainval_io.MAX_NORMALIZED_VALUE_KEY],
normalization_table=radar_normalization_table)
if len(matrix_dimensions) == 4:
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if list_of_layer_operation_dicts is None:
radar_field_names = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
radar_heights_m_agl = training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]
for j in range(len(radar_field_names)):
this_key = (radar_field_names[j], radar_heights_m_agl[j])
initial_matrix[..., j] = radar_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
else:
radar_field_names = [
d[input_examples.RADAR_FIELD_KEY]
for d in list_of_layer_operation_dicts
]
min_heights_m_agl = numpy.array([
d[input_examples.MIN_HEIGHT_KEY]
for d in list_of_layer_operation_dicts
],
dtype=int)
max_heights_m_agl = numpy.array([
d[input_examples.MAX_HEIGHT_KEY]
for d in list_of_layer_operation_dicts
],
dtype=int)
for j in range(len(radar_field_names)):
this_key = (radar_field_names[j], min_heights_m_agl[j])
this_first_mean = radar_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
this_key = (radar_field_names[j], max_heights_m_agl[j])
this_second_mean = radar_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
initial_matrix[..., j] = numpy.mean(
[this_first_mean, this_second_mean])
return dl_utils.normalize_radar_images(
radar_image_matrix=initial_matrix,
field_names=radar_field_names,
normalization_type_string=training_option_dict[
trainval_io.NORMALIZATION_TYPE_KEY],
normalization_param_file_name=training_option_dict[
trainval_io.NORMALIZATION_FILE_KEY],
test_mode=test_mode,
min_normalized_value=training_option_dict[
trainval_io.MIN_NORMALIZED_VALUE_KEY],
max_normalized_value=training_option_dict[
trainval_io.MAX_NORMALIZED_VALUE_KEY],
normalization_table=radar_normalization_table)
if len(matrix_dimensions) == 3:
sounding_field_names = training_option_dict[
trainval_io.SOUNDING_FIELDS_KEY]
sounding_heights_m_agl = training_option_dict[
trainval_io.SOUNDING_HEIGHTS_KEY]
for j in range(len(sounding_field_names)):
for k in range(len(sounding_heights_m_agl)):
this_key = (sounding_field_names[j],
sounding_heights_m_agl[k])
initial_matrix[..., k, j] = sounding_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
return dl_utils.normalize_soundings(
sounding_matrix=initial_matrix,
field_names=sounding_field_names,
normalization_type_string=training_option_dict[
trainval_io.NORMALIZATION_TYPE_KEY],
normalization_param_file_name=training_option_dict[
trainval_io.NORMALIZATION_FILE_KEY],
test_mode=test_mode,
min_normalized_value=training_option_dict[
trainval_io.MIN_NORMALIZED_VALUE_KEY],
max_normalized_value=training_option_dict[
trainval_io.MAX_NORMALIZED_VALUE_KEY],
normalization_table=sounding_normalization_table)
return None
def gridrad_generator_2d_reduced(option_dict, list_of_operation_dicts,
num_examples_total):
"""Generates examples with 2-D GridRad images.
These 2-D images are produced by applying layer operations to the native 3-D
images. The layer operations are specified by `list_of_operation_dicts`.
Each example (storm object) consists of the following:
- Storm-centered radar images (one 2-D image for each layer operation)
- Storm-centered sounding (optional)
- Target value (class)
:param option_dict: Dictionary with the following keys.
option_dict['example_file_names']: See doc for
`training_validation_io.gridrad_generator_2d_reduced`.
option_dict['binarize_target']: Same.
option_dict['sounding_field_names']: Same.
option_dict['sounding_heights_m_agl']: Same.
option_dict['first_storm_time_unix_sec']: Same.
option_dict['last_storm_time_unix_sec']: Same.
option_dict['num_grid_rows']: Same.
option_dict['num_grid_columns']: Same.
option_dict['normalization_type_string']: Same.
option_dict['normalization_param_file_name']: Same.
option_dict['min_normalized_value']: Same.
option_dict['max_normalized_value']: Same.
option_dict['class_to_sampling_fraction_dict']: Same.
:param list_of_operation_dicts: See doc for
`input_examples.reduce_examples_3d_to_2d`.
:param num_examples_total: Number of examples to generate.
:return: storm_object_dict: Dictionary with the following keys.
storm_object_dict['list_of_input_matrices']: length-T list of numpy arrays,
where T = number of input tensors to model. The first axis of each
array has length E.
storm_object_dict['storm_ids']: length-E list of storm IDs.
storm_object_dict['storm_times_unix_sec']: length-E numpy array of storm
times.
storm_object_dict['target_array']: See output doc for
`training_validation_io.gridrad_generator_2d_reduced`.
storm_object_dict['sounding_pressure_matrix_pascals']: numpy array (E x H_s)
of pressures. If soundings were not read, this is None.
storm_object_dict['radar_field_names']: length-C list of field names, where
the [j]th item corresponds to the [j]th channel of the 2-D radar images
returned in "list_of_input_matrices".
storm_object_dict['min_radar_heights_m_agl']: length-C numpy array with
minimum height for each layer operation (used to reduce 3-D radar images
to 2-D).
storm_object_dict['max_radar_heights_m_agl']: Same but with max heights.
storm_object_dict['radar_layer_operation_names']: length-C list with names
of layer operations. Each name must be accepted by
`input_examples._check_layer_operation`.
"""
unique_radar_field_names, unique_radar_heights_m_agl = (
trainval_io.layer_ops_to_field_height_pairs(list_of_operation_dicts)
)
option_dict[trainval_io.RADAR_FIELDS_KEY] = unique_radar_field_names
option_dict[trainval_io.RADAR_HEIGHTS_KEY] = unique_radar_heights_m_agl
storm_ids, storm_times_unix_sec = _find_examples_to_read(
option_dict=option_dict, num_examples_total=num_examples_total)
print '\n'
example_file_names = option_dict[trainval_io.EXAMPLE_FILES_KEY]
first_storm_time_unix_sec = option_dict[trainval_io.FIRST_STORM_TIME_KEY]
last_storm_time_unix_sec = option_dict[trainval_io.LAST_STORM_TIME_KEY]
num_grid_rows = option_dict[trainval_io.NUM_ROWS_KEY]
num_grid_columns = option_dict[trainval_io.NUM_COLUMNS_KEY]
sounding_field_names = option_dict[trainval_io.SOUNDING_FIELDS_KEY]
sounding_heights_m_agl = option_dict[trainval_io.SOUNDING_HEIGHTS_KEY]
normalization_type_string = option_dict[trainval_io.NORMALIZATION_TYPE_KEY]
normalization_param_file_name = option_dict[
trainval_io.NORMALIZATION_FILE_KEY]
min_normalized_value = option_dict[trainval_io.MIN_NORMALIZED_VALUE_KEY]
max_normalized_value = option_dict[trainval_io.MAX_NORMALIZED_VALUE_KEY]
binarize_target = option_dict[trainval_io.BINARIZE_TARGET_KEY]
this_example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_names[0], metadata_only=True)
target_name = this_example_dict[input_examples.TARGET_NAME_KEY]
num_classes = target_val_utils.target_name_to_num_classes(
target_name=target_name, include_dead_storms=False)
if sounding_field_names is None:
sounding_field_names_to_read = None
else:
if soundings.PRESSURE_NAME in sounding_field_names:
sounding_field_names_to_read = sounding_field_names + []
else:
sounding_field_names_to_read = (
sounding_field_names + [soundings.PRESSURE_NAME]
)
radar_image_matrix = None
sounding_matrix = None
target_values = None
sounding_pressure_matrix_pascals = None
reduction_metadata_dict = {}
file_index = 0
while True:
if file_index >= len(example_file_names):
raise StopIteration
print 'Reading data from: "{0:s}"...'.format(
example_file_names[file_index])
this_example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_names[file_index],
include_soundings=sounding_field_names is not None,
radar_field_names_to_keep=unique_radar_field_names,
radar_heights_to_keep_m_agl=unique_radar_heights_m_agl,
sounding_field_names_to_keep=sounding_field_names_to_read,
sounding_heights_to_keep_m_agl=sounding_heights_m_agl,
first_time_to_keep_unix_sec=first_storm_time_unix_sec,
last_time_to_keep_unix_sec=last_storm_time_unix_sec,
num_rows_to_keep=num_grid_rows,
num_columns_to_keep=num_grid_columns)
file_index += 1
if this_example_dict is None:
continue
indices_to_keep = tracking_utils.find_storm_objects(
all_storm_ids=this_example_dict[input_examples.STORM_IDS_KEY],
all_times_unix_sec=this_example_dict[
input_examples.STORM_TIMES_KEY],
storm_ids_to_keep=storm_ids,
times_to_keep_unix_sec=storm_times_unix_sec, allow_missing=True)
indices_to_keep = indices_to_keep[indices_to_keep >= 0]
if len(indices_to_keep) == 0:
continue
this_example_dict = input_examples.subset_examples(
example_dict=this_example_dict, indices_to_keep=indices_to_keep)
this_example_dict = input_examples.reduce_examples_3d_to_2d(
example_dict=this_example_dict,
list_of_operation_dicts=list_of_operation_dicts)
radar_field_names_2d = this_example_dict[
input_examples.RADAR_FIELDS_KEY]
for this_key in REDUCTION_METADATA_KEYS:
reduction_metadata_dict[this_key] = this_example_dict[this_key]
include_soundings = (
input_examples.SOUNDING_MATRIX_KEY in this_example_dict)
if include_soundings:
pressure_index = this_example_dict[
input_examples.SOUNDING_FIELDS_KEY
].index(soundings.PRESSURE_NAME)
this_pressure_matrix_pascals = this_example_dict[
input_examples.SOUNDING_MATRIX_KEY][..., pressure_index]
this_sounding_matrix = this_example_dict[
input_examples.SOUNDING_MATRIX_KEY]
if soundings.PRESSURE_NAME not in sounding_field_names:
this_sounding_matrix = this_sounding_matrix[..., :-1]
if target_values is None:
radar_image_matrix = (
this_example_dict[input_examples.RADAR_IMAGE_MATRIX_KEY]
+ 0.
)
target_values = (
this_example_dict[input_examples.TARGET_VALUES_KEY] + 0)
if include_soundings:
sounding_matrix = this_sounding_matrix + 0.
sounding_pressure_matrix_pascals = (
this_pressure_matrix_pascals + 0.)
else:
radar_image_matrix = numpy.concatenate(
(radar_image_matrix,
this_example_dict[input_examples.RADAR_IMAGE_MATRIX_KEY]),
axis=0)
target_values = numpy.concatenate((
target_values,
this_example_dict[input_examples.TARGET_VALUES_KEY]
))
if include_soundings:
sounding_matrix = numpy.concatenate(
(sounding_matrix, this_sounding_matrix), axis=0)
sounding_pressure_matrix_pascals = numpy.concatenate(
(sounding_pressure_matrix_pascals,
this_pressure_matrix_pascals), axis=0)
if normalization_type_string is not None:
radar_image_matrix = dl_utils.normalize_radar_images(
radar_image_matrix=radar_image_matrix,
field_names=radar_field_names_2d,
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
if include_soundings:
sounding_matrix = dl_utils.normalize_soundings(
sounding_matrix=sounding_matrix,
field_names=sounding_field_names,
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
list_of_predictor_matrices = [radar_image_matrix]
if include_soundings:
list_of_predictor_matrices.append(sounding_matrix)
target_array = _finalize_targets(
target_values=target_values, binarize_target=binarize_target,
num_classes=num_classes)
storm_object_dict = {
INPUT_MATRICES_KEY: list_of_predictor_matrices,
TARGET_ARRAY_KEY: target_array,
STORM_IDS_KEY: this_example_dict[input_examples.STORM_IDS_KEY],
STORM_TIMES_KEY: this_example_dict[input_examples.STORM_TIMES_KEY],
SOUNDING_PRESSURES_KEY:
copy.deepcopy(sounding_pressure_matrix_pascals)
}
for this_key in REDUCTION_METADATA_KEYS:
storm_object_dict[this_key] = reduction_metadata_dict[this_key]
radar_image_matrix = None
sounding_matrix = None
target_values = None
sounding_pressure_matrix_pascals = None
yield storm_object_dict
def myrorss_generator_2d3d(option_dict, num_examples_total):
"""Generates examples with both 2-D and 3-D radar images.
Each example (storm object) consists of the following:
- Storm-centered azimuthal shear (one 2-D image for each field)
- Storm-centered reflectivity (one 3-D image)
- Storm-centered sounding (optional)
- Target value (class)
:param option_dict: Dictionary with the following keys.
option_dict['example_file_names']: See doc for
`training_validation_io.myrorss_generator_2d3d`.
option_dict['binarize_target']: Same.
option_dict['radar_field_names']: Same.
option_dict['radar_heights_m_agl']: Same.
option_dict['sounding_field_names']: Same.
option_dict['sounding_heights_m_agl']: Same.
option_dict['first_storm_time_unix_sec']: Same.
option_dict['last_storm_time_unix_sec']: Same.
option_dict['num_grid_rows']: Same.
option_dict['num_grid_columns']: Same.
option_dict['normalization_type_string']: See doc for `generator_2d_or_3d`.
option_dict['normalization_param_file_name']: Same.
option_dict['min_normalized_value']: Same.
option_dict['max_normalized_value']: Same.
option_dict['class_to_sampling_fraction_dict']: Same.
:param num_examples_total: Total number of examples to generate.
:return: storm_object_dict: Dictionary with the following keys.
storm_object_dict['list_of_input_matrices']: length-T list of numpy arrays,
where T = number of input tensors to model. The first axis of each
array has length E.
storm_object_dict['storm_ids']: length-E list of storm IDs.
storm_object_dict['storm_times_unix_sec']: length-E numpy array of storm
times.
storm_object_dict['target_array']: See output doc for
`training_validation_io.myrorss_generator_2d3d`.
storm_object_dict['sounding_pressure_matrix_pascals']: numpy array (E x H_s)
of pressures. If soundings were not read, this is None.
"""
storm_ids, storm_times_unix_sec = _find_examples_to_read(
option_dict=option_dict, num_examples_total=num_examples_total)
print '\n'
example_file_names = option_dict[trainval_io.EXAMPLE_FILES_KEY]
first_storm_time_unix_sec = option_dict[trainval_io.FIRST_STORM_TIME_KEY]
last_storm_time_unix_sec = option_dict[trainval_io.LAST_STORM_TIME_KEY]
num_grid_rows = option_dict[trainval_io.NUM_ROWS_KEY]
num_grid_columns = option_dict[trainval_io.NUM_COLUMNS_KEY]
azimuthal_shear_field_names = option_dict[trainval_io.RADAR_FIELDS_KEY]
reflectivity_heights_m_agl = option_dict[trainval_io.RADAR_HEIGHTS_KEY]
sounding_field_names = option_dict[trainval_io.SOUNDING_FIELDS_KEY]
sounding_heights_m_agl = option_dict[trainval_io.SOUNDING_HEIGHTS_KEY]
normalization_type_string = option_dict[trainval_io.NORMALIZATION_TYPE_KEY]
normalization_param_file_name = option_dict[
trainval_io.NORMALIZATION_FILE_KEY]
min_normalized_value = option_dict[trainval_io.MIN_NORMALIZED_VALUE_KEY]
max_normalized_value = option_dict[trainval_io.MAX_NORMALIZED_VALUE_KEY]
binarize_target = option_dict[trainval_io.BINARIZE_TARGET_KEY]
this_example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_names[0], metadata_only=True)
target_name = this_example_dict[input_examples.TARGET_NAME_KEY]
num_classes = target_val_utils.target_name_to_num_classes(
target_name=target_name, include_dead_storms=False)
if sounding_field_names is None:
sounding_field_names_to_read = None
else:
if soundings.PRESSURE_NAME in sounding_field_names:
sounding_field_names_to_read = sounding_field_names + []
else:
sounding_field_names_to_read = (
sounding_field_names + [soundings.PRESSURE_NAME]
)
reflectivity_image_matrix_dbz = None
az_shear_image_matrix_s01 = None
sounding_matrix = None
target_values = None
sounding_pressure_matrix_pascals = None
file_index = 0
while True:
if file_index >= len(example_file_names):
raise StopIteration
print 'Reading data from: "{0:s}"...'.format(
example_file_names[file_index])
this_example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_names[file_index],
include_soundings=sounding_field_names is not None,
radar_field_names_to_keep=azimuthal_shear_field_names,
radar_heights_to_keep_m_agl=reflectivity_heights_m_agl,
sounding_field_names_to_keep=sounding_field_names_to_read,
sounding_heights_to_keep_m_agl=sounding_heights_m_agl,
first_time_to_keep_unix_sec=first_storm_time_unix_sec,
last_time_to_keep_unix_sec=last_storm_time_unix_sec,
num_rows_to_keep=num_grid_rows,
num_columns_to_keep=num_grid_columns)
file_index += 1
if this_example_dict is None:
continue
indices_to_keep = tracking_utils.find_storm_objects(
all_storm_ids=this_example_dict[input_examples.STORM_IDS_KEY],
all_times_unix_sec=this_example_dict[
input_examples.STORM_TIMES_KEY],
storm_ids_to_keep=storm_ids,
times_to_keep_unix_sec=storm_times_unix_sec, allow_missing=True)
indices_to_keep = indices_to_keep[indices_to_keep >= 0]
if len(indices_to_keep) == 0:
continue
this_example_dict = input_examples.subset_examples(
example_dict=this_example_dict, indices_to_keep=indices_to_keep)
include_soundings = (
input_examples.SOUNDING_MATRIX_KEY in this_example_dict)
if include_soundings:
pressure_index = this_example_dict[
input_examples.SOUNDING_FIELDS_KEY
].index(soundings.PRESSURE_NAME)
this_pressure_matrix_pascals = this_example_dict[
input_examples.SOUNDING_MATRIX_KEY][..., pressure_index]
this_sounding_matrix = this_example_dict[
input_examples.SOUNDING_MATRIX_KEY]
if soundings.PRESSURE_NAME not in sounding_field_names:
this_sounding_matrix = this_sounding_matrix[..., -1]
if target_values is None:
reflectivity_image_matrix_dbz = (
this_example_dict[input_examples.REFL_IMAGE_MATRIX_KEY] + 0.
)
az_shear_image_matrix_s01 = (
this_example_dict[input_examples.AZ_SHEAR_IMAGE_MATRIX_KEY]
+ 0.
)
target_values = (
this_example_dict[input_examples.TARGET_VALUES_KEY] + 0)
if include_soundings:
sounding_matrix = this_sounding_matrix + 0.
sounding_pressure_matrix_pascals = (
this_pressure_matrix_pascals + 0.)
else:
reflectivity_image_matrix_dbz = numpy.concatenate(
(reflectivity_image_matrix_dbz,
this_example_dict[input_examples.REFL_IMAGE_MATRIX_KEY]),
axis=0)
az_shear_image_matrix_s01 = numpy.concatenate((
az_shear_image_matrix_s01,
this_example_dict[input_examples.AZ_SHEAR_IMAGE_MATRIX_KEY]
), axis=0)
target_values = numpy.concatenate((
target_values,
this_example_dict[input_examples.TARGET_VALUES_KEY]
))
if include_soundings:
sounding_matrix = numpy.concatenate(
(sounding_matrix, this_sounding_matrix), axis=0)
sounding_pressure_matrix_pascals = numpy.concatenate(
(sounding_pressure_matrix_pascals,
this_pressure_matrix_pascals), axis=0)
if normalization_type_string is not None:
reflectivity_image_matrix_dbz = dl_utils.normalize_radar_images(
radar_image_matrix=reflectivity_image_matrix_dbz,
field_names=[radar_utils.REFL_NAME],
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
az_shear_image_matrix_s01 = dl_utils.normalize_radar_images(
radar_image_matrix=az_shear_image_matrix_s01,
field_names=azimuthal_shear_field_names,
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
if include_soundings:
sounding_matrix = dl_utils.normalize_soundings(
sounding_matrix=sounding_matrix,
field_names=sounding_field_names,
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
list_of_predictor_matrices = [
reflectivity_image_matrix_dbz, az_shear_image_matrix_s01
]
if include_soundings:
list_of_predictor_matrices.append(sounding_matrix)
target_array = _finalize_targets(
target_values=target_values, binarize_target=binarize_target,
num_classes=num_classes)
storm_object_dict = {
INPUT_MATRICES_KEY: list_of_predictor_matrices,
TARGET_ARRAY_KEY: target_array,
STORM_IDS_KEY: this_example_dict[input_examples.STORM_IDS_KEY],
STORM_TIMES_KEY: this_example_dict[input_examples.STORM_TIMES_KEY],
SOUNDING_PRESSURES_KEY: sounding_pressure_matrix_pascals + 0.
}
reflectivity_image_matrix_dbz = None
az_shear_image_matrix_s01 = None
sounding_matrix = None
target_values = None
sounding_pressure_matrix_pascals = None
yield storm_object_dict
def __normalize_minmax_for_sounding_field(field_name,
cnn_metadata_dict,
test_mode=False,
normalization_table=None):
"""Converts min and max for sounding field to normalized units.
:param field_name: Name of sounding field (must be accepted by
`soundings.check_field_name`).
:param cnn_metadata_dict: See doc for `_normalize_minima_and_maxima`.
:param test_mode: Same.
:param normalization_table: Same.
:return: min_normalized_value: Minimum value in normalized units.
:return: max_normalized_value: Max value in normalized units.
"""
training_option_dict = cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
normalization_type_string = training_option_dict[
trainval_io.NORMALIZATION_TYPE_KEY]
min_normalized_value = training_option_dict[
trainval_io.MIN_NORMALIZED_VALUE_KEY]
max_normalized_value = training_option_dict[
trainval_io.MAX_NORMALIZED_VALUE_KEY]
if test_mode:
normalization_file_name = None
else:
normalization_file_name = training_option_dict[
trainval_io.NORMALIZATION_FILE_KEY]
if field_name in SOUNDING_TO_MINIMUM_DICT:
if normalization_type_string == dl_utils.Z_NORMALIZATION_TYPE_STRING:
min_physical_value = SOUNDING_TO_MINIMUM_DICT[field_name]
dummy_sounding_matrix = numpy.full((1, 1, 1), min_physical_value)
dummy_sounding_matrix = dl_utils.normalize_soundings(
sounding_matrix=dummy_sounding_matrix,
field_names=[field_name],
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value,
test_mode=test_mode,
normalization_table=normalization_table)
min_normalized_value = numpy.ravel(dummy_sounding_matrix)[0]
else:
min_normalized_value = numpy.nan
if field_name in SOUNDING_TO_MAX_DICT:
if normalization_type_string == dl_utils.Z_NORMALIZATION_TYPE_STRING:
max_physical_value = SOUNDING_TO_MAX_DICT[field_name]
dummy_sounding_matrix = numpy.full((1, 1, 1), max_physical_value)
dummy_sounding_matrix = dl_utils.normalize_soundings(
sounding_matrix=dummy_sounding_matrix,
field_names=[field_name],
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value,
test_mode=test_mode,
normalization_table=normalization_table)
max_normalized_value = numpy.ravel(dummy_sounding_matrix)[0]
else:
max_normalized_value = numpy.nan
return min_normalized_value, max_normalized_value