def test_normalize_radar_images_5d_z(self):
"""Ensures correct output from normalize_radar_images.
In this case, the input matrix is 5-D and normalization type is z-score.
"""
this_radar_matrix = dl_utils.normalize_radar_images(
radar_image_matrix=copy.deepcopy(RADAR_MATRIX_5D_UNNORMALIZED),
field_names=RADAR_FIELD_NAMES,
normalization_type_string=dl_utils.Z_NORMALIZATION_TYPE_STRING,
normalization_param_file_name=None,
test_mode=True,
normalization_table=RADAR_NORMALIZATION_TABLE)
self.assertTrue(
numpy.allclose(this_radar_matrix,
RADAR_MATRIX_5D_Z_SCORES,
atol=TOLERANCE,
equal_nan=True))
def test_normalize_radar_images_4d_minmax(self):
"""Ensures correct output from normalize_radar_images.
In this case, the input matrix is 4-D and normalization type is minmax.
"""
this_radar_matrix = dl_utils.normalize_radar_images(
radar_image_matrix=copy.deepcopy(RADAR_MATRIX_4D_UNNORMALIZED),
field_names=RADAR_FIELD_NAMES,
normalization_type_string=dl_utils.
MINMAX_NORMALIZATION_TYPE_STRING,
normalization_param_file_name=None,
test_mode=True,
min_normalized_value=MIN_NORMALIZED_VALUE,
max_normalized_value=MAX_NORMALIZED_VALUE,
normalization_table=RADAR_NORMALIZATION_TABLE)
self.assertTrue(
numpy.allclose(this_radar_matrix,
RADAR_MATRIX_4D_MINMAX,
atol=TOLERANCE,
equal_nan=True))
def init_function(matrix_dimensions):
"""Initializes model input to climatological means.
This function uses one mean for each radar field/height pair and each
sounding field/height pair, rather than one per field altogether, to
create "realistic" vertical profiles.
If len(matrix_dimensions) = 3, this function creates initial soundings.
If len(matrix_dimensions) = 4 and myrorss_2d3d = False, creates initial
2-D radar images with all fields in `training_option_dict`.
If len(matrix_dimensions) = 4 and myrorss_2d3d = True, creates initial
2-D radar images with only azimuthal-shear fields in
`training_option_dict`.
If len(matrix_dimensions) = 5 and myrorss_2d3d = False, creates initial
3-D radar images with all fields in `training_option_dict`.
If len(matrix_dimensions) = 5 and myrorss_2d3d = True, creates initial
3-D reflectivity images.
:param matrix_dimensions: numpy array with desired dimensions
for initialized model input.
:return: initial_matrix: numpy array with the given dimensions.
"""
initial_matrix = numpy.full(matrix_dimensions, numpy.nan)
if len(matrix_dimensions) == 5:
if model_metadata_dict[cnn.USE_2D3D_CONVOLUTION_KEY]:
radar_field_names = [radar_utils.REFL_NAME]
else:
radar_field_names = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
radar_heights_m_agl = training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]
for j in range(len(radar_field_names)):
for k in range(len(radar_heights_m_agl)):
this_key = (radar_field_names[j], radar_heights_m_agl[k])
initial_matrix[..., k, j] = radar_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
return dl_utils.normalize_radar_images(
radar_image_matrix=initial_matrix,
field_names=radar_field_names,
normalization_type_string=training_option_dict[
trainval_io.NORMALIZATION_TYPE_KEY],
normalization_param_file_name=training_option_dict[
trainval_io.NORMALIZATION_FILE_KEY],
test_mode=test_mode,
min_normalized_value=training_option_dict[
trainval_io.MIN_NORMALIZED_VALUE_KEY],
max_normalized_value=training_option_dict[
trainval_io.MAX_NORMALIZED_VALUE_KEY],
normalization_table=radar_normalization_table)
if len(matrix_dimensions) == 4:
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if list_of_layer_operation_dicts is None:
radar_field_names = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
radar_heights_m_agl = training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]
for j in range(len(radar_field_names)):
this_key = (radar_field_names[j], radar_heights_m_agl[j])
initial_matrix[..., j] = radar_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
else:
radar_field_names = [
d[input_examples.RADAR_FIELD_KEY]
for d in list_of_layer_operation_dicts
]
min_heights_m_agl = numpy.array([
d[input_examples.MIN_HEIGHT_KEY]
for d in list_of_layer_operation_dicts
],
dtype=int)
max_heights_m_agl = numpy.array([
d[input_examples.MAX_HEIGHT_KEY]
for d in list_of_layer_operation_dicts
],
dtype=int)
for j in range(len(radar_field_names)):
this_key = (radar_field_names[j], min_heights_m_agl[j])
this_first_mean = radar_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
this_key = (radar_field_names[j], max_heights_m_agl[j])
this_second_mean = radar_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
initial_matrix[..., j] = numpy.mean(
[this_first_mean, this_second_mean])
return dl_utils.normalize_radar_images(
radar_image_matrix=initial_matrix,
field_names=radar_field_names,
normalization_type_string=training_option_dict[
trainval_io.NORMALIZATION_TYPE_KEY],
normalization_param_file_name=training_option_dict[
trainval_io.NORMALIZATION_FILE_KEY],
test_mode=test_mode,
min_normalized_value=training_option_dict[
trainval_io.MIN_NORMALIZED_VALUE_KEY],
max_normalized_value=training_option_dict[
trainval_io.MAX_NORMALIZED_VALUE_KEY],
normalization_table=radar_normalization_table)
if len(matrix_dimensions) == 3:
sounding_field_names = training_option_dict[
trainval_io.SOUNDING_FIELDS_KEY]
sounding_heights_m_agl = training_option_dict[
trainval_io.SOUNDING_HEIGHTS_KEY]
for j in range(len(sounding_field_names)):
for k in range(len(sounding_heights_m_agl)):
this_key = (sounding_field_names[j],
sounding_heights_m_agl[k])
initial_matrix[..., k, j] = sounding_normalization_table[
dl_utils.MEAN_VALUE_COLUMN].loc[[this_key]].values[0]
return dl_utils.normalize_soundings(
sounding_matrix=initial_matrix,
field_names=sounding_field_names,
normalization_type_string=training_option_dict[
trainval_io.NORMALIZATION_TYPE_KEY],
normalization_param_file_name=training_option_dict[
trainval_io.NORMALIZATION_FILE_KEY],
test_mode=test_mode,
min_normalized_value=training_option_dict[
trainval_io.MIN_NORMALIZED_VALUE_KEY],
max_normalized_value=training_option_dict[
trainval_io.MAX_NORMALIZED_VALUE_KEY],
normalization_table=sounding_normalization_table)
return None
def gridrad_generator_2d_reduced(option_dict, list_of_operation_dicts,
num_examples_total):
"""Generates examples with 2-D GridRad images.
These 2-D images are produced by applying layer operations to the native 3-D
images. The layer operations are specified by `list_of_operation_dicts`.
Each example (storm object) consists of the following:
- Storm-centered radar images (one 2-D image for each layer operation)
- Storm-centered sounding (optional)
- Target value (class)
:param option_dict: Dictionary with the following keys.
option_dict['example_file_names']: See doc for
`training_validation_io.gridrad_generator_2d_reduced`.
option_dict['binarize_target']: Same.
option_dict['sounding_field_names']: Same.
option_dict['sounding_heights_m_agl']: Same.
option_dict['first_storm_time_unix_sec']: Same.
option_dict['last_storm_time_unix_sec']: Same.
option_dict['num_grid_rows']: Same.
option_dict['num_grid_columns']: Same.
option_dict['normalization_type_string']: Same.
option_dict['normalization_param_file_name']: Same.
option_dict['min_normalized_value']: Same.
option_dict['max_normalized_value']: Same.
option_dict['class_to_sampling_fraction_dict']: Same.
:param list_of_operation_dicts: See doc for
`input_examples.reduce_examples_3d_to_2d`.
:param num_examples_total: Number of examples to generate.
:return: storm_object_dict: Dictionary with the following keys.
storm_object_dict['list_of_input_matrices']: length-T list of numpy arrays,
where T = number of input tensors to model. The first axis of each
array has length E.
storm_object_dict['storm_ids']: length-E list of storm IDs.
storm_object_dict['storm_times_unix_sec']: length-E numpy array of storm
times.
storm_object_dict['target_array']: See output doc for
`training_validation_io.gridrad_generator_2d_reduced`.
storm_object_dict['sounding_pressure_matrix_pascals']: numpy array (E x H_s)
of pressures. If soundings were not read, this is None.
storm_object_dict['radar_field_names']: length-C list of field names, where
the [j]th item corresponds to the [j]th channel of the 2-D radar images
returned in "list_of_input_matrices".
storm_object_dict['min_radar_heights_m_agl']: length-C numpy array with
minimum height for each layer operation (used to reduce 3-D radar images
to 2-D).
storm_object_dict['max_radar_heights_m_agl']: Same but with max heights.
storm_object_dict['radar_layer_operation_names']: length-C list with names
of layer operations. Each name must be accepted by
`input_examples._check_layer_operation`.
"""
unique_radar_field_names, unique_radar_heights_m_agl = (
trainval_io.layer_ops_to_field_height_pairs(list_of_operation_dicts)
)
option_dict[trainval_io.RADAR_FIELDS_KEY] = unique_radar_field_names
option_dict[trainval_io.RADAR_HEIGHTS_KEY] = unique_radar_heights_m_agl
storm_ids, storm_times_unix_sec = _find_examples_to_read(
option_dict=option_dict, num_examples_total=num_examples_total)
print '\n'
example_file_names = option_dict[trainval_io.EXAMPLE_FILES_KEY]
first_storm_time_unix_sec = option_dict[trainval_io.FIRST_STORM_TIME_KEY]
last_storm_time_unix_sec = option_dict[trainval_io.LAST_STORM_TIME_KEY]
num_grid_rows = option_dict[trainval_io.NUM_ROWS_KEY]
num_grid_columns = option_dict[trainval_io.NUM_COLUMNS_KEY]
sounding_field_names = option_dict[trainval_io.SOUNDING_FIELDS_KEY]
sounding_heights_m_agl = option_dict[trainval_io.SOUNDING_HEIGHTS_KEY]
normalization_type_string = option_dict[trainval_io.NORMALIZATION_TYPE_KEY]
normalization_param_file_name = option_dict[
trainval_io.NORMALIZATION_FILE_KEY]
min_normalized_value = option_dict[trainval_io.MIN_NORMALIZED_VALUE_KEY]
max_normalized_value = option_dict[trainval_io.MAX_NORMALIZED_VALUE_KEY]
binarize_target = option_dict[trainval_io.BINARIZE_TARGET_KEY]
this_example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_names[0], metadata_only=True)
target_name = this_example_dict[input_examples.TARGET_NAME_KEY]
num_classes = target_val_utils.target_name_to_num_classes(
target_name=target_name, include_dead_storms=False)
if sounding_field_names is None:
sounding_field_names_to_read = None
else:
if soundings.PRESSURE_NAME in sounding_field_names:
sounding_field_names_to_read = sounding_field_names + []
else:
sounding_field_names_to_read = (
sounding_field_names + [soundings.PRESSURE_NAME]
)
radar_image_matrix = None
sounding_matrix = None
target_values = None
sounding_pressure_matrix_pascals = None
reduction_metadata_dict = {}
file_index = 0
while True:
if file_index >= len(example_file_names):
raise StopIteration
print 'Reading data from: "{0:s}"...'.format(
example_file_names[file_index])
this_example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_names[file_index],
include_soundings=sounding_field_names is not None,
radar_field_names_to_keep=unique_radar_field_names,
radar_heights_to_keep_m_agl=unique_radar_heights_m_agl,
sounding_field_names_to_keep=sounding_field_names_to_read,
sounding_heights_to_keep_m_agl=sounding_heights_m_agl,
first_time_to_keep_unix_sec=first_storm_time_unix_sec,
last_time_to_keep_unix_sec=last_storm_time_unix_sec,
num_rows_to_keep=num_grid_rows,
num_columns_to_keep=num_grid_columns)
file_index += 1
if this_example_dict is None:
continue
indices_to_keep = tracking_utils.find_storm_objects(
all_storm_ids=this_example_dict[input_examples.STORM_IDS_KEY],
all_times_unix_sec=this_example_dict[
input_examples.STORM_TIMES_KEY],
storm_ids_to_keep=storm_ids,
times_to_keep_unix_sec=storm_times_unix_sec, allow_missing=True)
indices_to_keep = indices_to_keep[indices_to_keep >= 0]
if len(indices_to_keep) == 0:
continue
this_example_dict = input_examples.subset_examples(
example_dict=this_example_dict, indices_to_keep=indices_to_keep)
this_example_dict = input_examples.reduce_examples_3d_to_2d(
example_dict=this_example_dict,
list_of_operation_dicts=list_of_operation_dicts)
radar_field_names_2d = this_example_dict[
input_examples.RADAR_FIELDS_KEY]
for this_key in REDUCTION_METADATA_KEYS:
reduction_metadata_dict[this_key] = this_example_dict[this_key]
include_soundings = (
input_examples.SOUNDING_MATRIX_KEY in this_example_dict)
if include_soundings:
pressure_index = this_example_dict[
input_examples.SOUNDING_FIELDS_KEY
].index(soundings.PRESSURE_NAME)
this_pressure_matrix_pascals = this_example_dict[
input_examples.SOUNDING_MATRIX_KEY][..., pressure_index]
this_sounding_matrix = this_example_dict[
input_examples.SOUNDING_MATRIX_KEY]
if soundings.PRESSURE_NAME not in sounding_field_names:
this_sounding_matrix = this_sounding_matrix[..., :-1]
if target_values is None:
radar_image_matrix = (
this_example_dict[input_examples.RADAR_IMAGE_MATRIX_KEY]
+ 0.
)
target_values = (
this_example_dict[input_examples.TARGET_VALUES_KEY] + 0)
if include_soundings:
sounding_matrix = this_sounding_matrix + 0.
sounding_pressure_matrix_pascals = (
this_pressure_matrix_pascals + 0.)
else:
radar_image_matrix = numpy.concatenate(
(radar_image_matrix,
this_example_dict[input_examples.RADAR_IMAGE_MATRIX_KEY]),
axis=0)
target_values = numpy.concatenate((
target_values,
this_example_dict[input_examples.TARGET_VALUES_KEY]
))
if include_soundings:
sounding_matrix = numpy.concatenate(
(sounding_matrix, this_sounding_matrix), axis=0)
sounding_pressure_matrix_pascals = numpy.concatenate(
(sounding_pressure_matrix_pascals,
this_pressure_matrix_pascals), axis=0)
if normalization_type_string is not None:
radar_image_matrix = dl_utils.normalize_radar_images(
radar_image_matrix=radar_image_matrix,
field_names=radar_field_names_2d,
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
if include_soundings:
sounding_matrix = dl_utils.normalize_soundings(
sounding_matrix=sounding_matrix,
field_names=sounding_field_names,
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
list_of_predictor_matrices = [radar_image_matrix]
if include_soundings:
list_of_predictor_matrices.append(sounding_matrix)
target_array = _finalize_targets(
target_values=target_values, binarize_target=binarize_target,
num_classes=num_classes)
storm_object_dict = {
INPUT_MATRICES_KEY: list_of_predictor_matrices,
TARGET_ARRAY_KEY: target_array,
STORM_IDS_KEY: this_example_dict[input_examples.STORM_IDS_KEY],
STORM_TIMES_KEY: this_example_dict[input_examples.STORM_TIMES_KEY],
SOUNDING_PRESSURES_KEY:
copy.deepcopy(sounding_pressure_matrix_pascals)
}
for this_key in REDUCTION_METADATA_KEYS:
storm_object_dict[this_key] = reduction_metadata_dict[this_key]
radar_image_matrix = None
sounding_matrix = None
target_values = None
sounding_pressure_matrix_pascals = None
yield storm_object_dict
def myrorss_generator_2d3d(option_dict, num_examples_total):
"""Generates examples with both 2-D and 3-D radar images.
Each example (storm object) consists of the following:
- Storm-centered azimuthal shear (one 2-D image for each field)
- Storm-centered reflectivity (one 3-D image)
- Storm-centered sounding (optional)
- Target value (class)
:param option_dict: Dictionary with the following keys.
option_dict['example_file_names']: See doc for
`training_validation_io.myrorss_generator_2d3d`.
option_dict['binarize_target']: Same.
option_dict['radar_field_names']: Same.
option_dict['radar_heights_m_agl']: Same.
option_dict['sounding_field_names']: Same.
option_dict['sounding_heights_m_agl']: Same.
option_dict['first_storm_time_unix_sec']: Same.
option_dict['last_storm_time_unix_sec']: Same.
option_dict['num_grid_rows']: Same.
option_dict['num_grid_columns']: Same.
option_dict['normalization_type_string']: See doc for `generator_2d_or_3d`.
option_dict['normalization_param_file_name']: Same.
option_dict['min_normalized_value']: Same.
option_dict['max_normalized_value']: Same.
option_dict['class_to_sampling_fraction_dict']: Same.
:param num_examples_total: Total number of examples to generate.
:return: storm_object_dict: Dictionary with the following keys.
storm_object_dict['list_of_input_matrices']: length-T list of numpy arrays,
where T = number of input tensors to model. The first axis of each
array has length E.
storm_object_dict['storm_ids']: length-E list of storm IDs.
storm_object_dict['storm_times_unix_sec']: length-E numpy array of storm
times.
storm_object_dict['target_array']: See output doc for
`training_validation_io.myrorss_generator_2d3d`.
storm_object_dict['sounding_pressure_matrix_pascals']: numpy array (E x H_s)
of pressures. If soundings were not read, this is None.
"""
storm_ids, storm_times_unix_sec = _find_examples_to_read(
option_dict=option_dict, num_examples_total=num_examples_total)
print '\n'
example_file_names = option_dict[trainval_io.EXAMPLE_FILES_KEY]
first_storm_time_unix_sec = option_dict[trainval_io.FIRST_STORM_TIME_KEY]
last_storm_time_unix_sec = option_dict[trainval_io.LAST_STORM_TIME_KEY]
num_grid_rows = option_dict[trainval_io.NUM_ROWS_KEY]
num_grid_columns = option_dict[trainval_io.NUM_COLUMNS_KEY]
azimuthal_shear_field_names = option_dict[trainval_io.RADAR_FIELDS_KEY]
reflectivity_heights_m_agl = option_dict[trainval_io.RADAR_HEIGHTS_KEY]
sounding_field_names = option_dict[trainval_io.SOUNDING_FIELDS_KEY]
sounding_heights_m_agl = option_dict[trainval_io.SOUNDING_HEIGHTS_KEY]
normalization_type_string = option_dict[trainval_io.NORMALIZATION_TYPE_KEY]
normalization_param_file_name = option_dict[
trainval_io.NORMALIZATION_FILE_KEY]
min_normalized_value = option_dict[trainval_io.MIN_NORMALIZED_VALUE_KEY]
max_normalized_value = option_dict[trainval_io.MAX_NORMALIZED_VALUE_KEY]
binarize_target = option_dict[trainval_io.BINARIZE_TARGET_KEY]
this_example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_names[0], metadata_only=True)
target_name = this_example_dict[input_examples.TARGET_NAME_KEY]
num_classes = target_val_utils.target_name_to_num_classes(
target_name=target_name, include_dead_storms=False)
if sounding_field_names is None:
sounding_field_names_to_read = None
else:
if soundings.PRESSURE_NAME in sounding_field_names:
sounding_field_names_to_read = sounding_field_names + []
else:
sounding_field_names_to_read = (
sounding_field_names + [soundings.PRESSURE_NAME]
)
reflectivity_image_matrix_dbz = None
az_shear_image_matrix_s01 = None
sounding_matrix = None
target_values = None
sounding_pressure_matrix_pascals = None
file_index = 0
while True:
if file_index >= len(example_file_names):
raise StopIteration
print 'Reading data from: "{0:s}"...'.format(
example_file_names[file_index])
this_example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_names[file_index],
include_soundings=sounding_field_names is not None,
radar_field_names_to_keep=azimuthal_shear_field_names,
radar_heights_to_keep_m_agl=reflectivity_heights_m_agl,
sounding_field_names_to_keep=sounding_field_names_to_read,
sounding_heights_to_keep_m_agl=sounding_heights_m_agl,
first_time_to_keep_unix_sec=first_storm_time_unix_sec,
last_time_to_keep_unix_sec=last_storm_time_unix_sec,
num_rows_to_keep=num_grid_rows,
num_columns_to_keep=num_grid_columns)
file_index += 1
if this_example_dict is None:
continue
indices_to_keep = tracking_utils.find_storm_objects(
all_storm_ids=this_example_dict[input_examples.STORM_IDS_KEY],
all_times_unix_sec=this_example_dict[
input_examples.STORM_TIMES_KEY],
storm_ids_to_keep=storm_ids,
times_to_keep_unix_sec=storm_times_unix_sec, allow_missing=True)
indices_to_keep = indices_to_keep[indices_to_keep >= 0]
if len(indices_to_keep) == 0:
continue
this_example_dict = input_examples.subset_examples(
example_dict=this_example_dict, indices_to_keep=indices_to_keep)
include_soundings = (
input_examples.SOUNDING_MATRIX_KEY in this_example_dict)
if include_soundings:
pressure_index = this_example_dict[
input_examples.SOUNDING_FIELDS_KEY
].index(soundings.PRESSURE_NAME)
this_pressure_matrix_pascals = this_example_dict[
input_examples.SOUNDING_MATRIX_KEY][..., pressure_index]
this_sounding_matrix = this_example_dict[
input_examples.SOUNDING_MATRIX_KEY]
if soundings.PRESSURE_NAME not in sounding_field_names:
this_sounding_matrix = this_sounding_matrix[..., -1]
if target_values is None:
reflectivity_image_matrix_dbz = (
this_example_dict[input_examples.REFL_IMAGE_MATRIX_KEY] + 0.
)
az_shear_image_matrix_s01 = (
this_example_dict[input_examples.AZ_SHEAR_IMAGE_MATRIX_KEY]
+ 0.
)
target_values = (
this_example_dict[input_examples.TARGET_VALUES_KEY] + 0)
if include_soundings:
sounding_matrix = this_sounding_matrix + 0.
sounding_pressure_matrix_pascals = (
this_pressure_matrix_pascals + 0.)
else:
reflectivity_image_matrix_dbz = numpy.concatenate(
(reflectivity_image_matrix_dbz,
this_example_dict[input_examples.REFL_IMAGE_MATRIX_KEY]),
axis=0)
az_shear_image_matrix_s01 = numpy.concatenate((
az_shear_image_matrix_s01,
this_example_dict[input_examples.AZ_SHEAR_IMAGE_MATRIX_KEY]
), axis=0)
target_values = numpy.concatenate((
target_values,
this_example_dict[input_examples.TARGET_VALUES_KEY]
))
if include_soundings:
sounding_matrix = numpy.concatenate(
(sounding_matrix, this_sounding_matrix), axis=0)
sounding_pressure_matrix_pascals = numpy.concatenate(
(sounding_pressure_matrix_pascals,
this_pressure_matrix_pascals), axis=0)
if normalization_type_string is not None:
reflectivity_image_matrix_dbz = dl_utils.normalize_radar_images(
radar_image_matrix=reflectivity_image_matrix_dbz,
field_names=[radar_utils.REFL_NAME],
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
az_shear_image_matrix_s01 = dl_utils.normalize_radar_images(
radar_image_matrix=az_shear_image_matrix_s01,
field_names=azimuthal_shear_field_names,
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
if include_soundings:
sounding_matrix = dl_utils.normalize_soundings(
sounding_matrix=sounding_matrix,
field_names=sounding_field_names,
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_param_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value).astype('float32')
list_of_predictor_matrices = [
reflectivity_image_matrix_dbz, az_shear_image_matrix_s01
]
if include_soundings:
list_of_predictor_matrices.append(sounding_matrix)
target_array = _finalize_targets(
target_values=target_values, binarize_target=binarize_target,
num_classes=num_classes)
storm_object_dict = {
INPUT_MATRICES_KEY: list_of_predictor_matrices,
TARGET_ARRAY_KEY: target_array,
STORM_IDS_KEY: this_example_dict[input_examples.STORM_IDS_KEY],
STORM_TIMES_KEY: this_example_dict[input_examples.STORM_TIMES_KEY],
SOUNDING_PRESSURES_KEY: sounding_pressure_matrix_pascals + 0.
}
reflectivity_image_matrix_dbz = None
az_shear_image_matrix_s01 = None
sounding_matrix = None
target_values = None
sounding_pressure_matrix_pascals = None
yield storm_object_dict
def _run(example_file_name, example_indices, num_radar_rows, num_radar_columns,
normalization_file_name, output_dir_name):
"""Makes figure to explain one convolution block.
This is effectively the main method.
:param example_file_name: See documentation at top of file.
:param example_indices: Same.
:param num_radar_rows: Same.
:param num_radar_columns: Same.
:param normalization_file_name: Same.
:param output_dir_name: Same.
"""
if num_radar_rows <= 0:
num_radar_rows = None
if num_radar_columns <= 0:
num_radar_columns = None
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
print('Reading data from: "{0:s}"...'.format(example_file_name))
example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_name, read_all_target_vars=False,
target_name=DUMMY_TARGET_NAME, include_soundings=False,
num_rows_to_keep=num_radar_rows, num_columns_to_keep=num_radar_columns,
radar_heights_to_keep_m_agl=numpy.array([RADAR_HEIGHT_M_AGL], dtype=int)
)
if input_examples.REFL_IMAGE_MATRIX_KEY in example_dict:
input_feature_matrix = example_dict[
input_examples.REFL_IMAGE_MATRIX_KEY]
else:
field_index = example_dict[input_examples.RADAR_FIELDS_KEY].index(
RADAR_FIELD_NAME
)
input_feature_matrix = example_dict[
input_examples.RADAR_IMAGE_MATRIX_KEY
][..., [field_index]]
num_examples = input_feature_matrix.shape[0]
error_checking.assert_is_geq_numpy_array(example_indices, 0)
error_checking.assert_is_less_than_numpy_array(
example_indices, num_examples)
input_feature_matrix = dl_utils.normalize_radar_images(
radar_image_matrix=input_feature_matrix, field_names=[RADAR_FIELD_NAME],
normalization_type_string=NORMALIZATION_TYPE_STRING,
normalization_param_file_name=normalization_file_name)
if len(input_feature_matrix.shape) == 4:
input_feature_matrix = input_feature_matrix[..., 0]
else:
input_feature_matrix = input_feature_matrix[..., 0, 0]
input_feature_matrix = numpy.expand_dims(input_feature_matrix, axis=-1)
print('Doing convolution for all {0:d} examples...'.format(num_examples))
feature_matrix_after_conv = None
for i in range(num_examples):
this_feature_matrix = standalone_utils.do_2d_convolution(
feature_matrix=input_feature_matrix[i, ...] + 0,
kernel_matrix=KERNEL_MATRIX, pad_edges=False, stride_length_px=1
)[0, ...]
if feature_matrix_after_conv is None:
feature_matrix_after_conv = numpy.full(
(num_examples,) + this_feature_matrix.shape, numpy.nan
)
feature_matrix_after_conv[i, ...] = this_feature_matrix
print('Doing activation for all {0:d} examples...'.format(num_examples))
feature_matrix_after_activn = standalone_utils.do_activation(
input_values=feature_matrix_after_conv + 0,
function_name=architecture_utils.RELU_FUNCTION_STRING, alpha=0.2)
print('Doing batch norm for all {0:d} examples...'.format(num_examples))
feature_matrix_after_bn = standalone_utils.do_batch_normalization(
feature_matrix=feature_matrix_after_activn + 0
)
print('Doing max-pooling for all {0:d} examples...\n'.format(num_examples))
feature_matrix_after_pooling = None
for i in range(num_examples):
this_feature_matrix = standalone_utils.do_2d_pooling(
feature_matrix=feature_matrix_after_bn[i, ...], stride_length_px=2,
pooling_type_string=standalone_utils.MAX_POOLING_TYPE_STRING
)[0, ...]
if feature_matrix_after_pooling is None:
feature_matrix_after_pooling = numpy.full(
(num_examples,) + this_feature_matrix.shape, numpy.nan
)
feature_matrix_after_pooling[i, ...] = this_feature_matrix
for i in example_indices:
this_output_file_name = '{0:s}/convolution_block{1:06d}.jpg'.format(
output_dir_name, i)
_plot_one_example(
input_feature_matrix=input_feature_matrix[i, ...],
feature_matrix_after_conv=feature_matrix_after_conv[i, ...],
feature_matrix_after_activn=feature_matrix_after_activn[i, ...],
feature_matrix_after_bn=feature_matrix_after_bn[i, ...],
feature_matrix_after_pooling=feature_matrix_after_pooling[i, ...],
output_file_name=this_output_file_name)
def __normalize_minmax_for_radar_field(field_name,
cnn_metadata_dict,
test_mode=False,
normalization_table=None):
"""Converts min and max for radar field to normalized units.
:param field_name: Name of radar field (must be accepted by
`radar_utils.check_field_name`).
:param cnn_metadata_dict: See doc for `_normalize_minima_and_maxima`.
:param test_mode: Same.
:param normalization_table: Same.
:return: min_normalized_value: Minimum value in normalized units.
:return: max_normalized_value: Max value in normalized units.
"""
training_option_dict = cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
normalization_type_string = training_option_dict[
trainval_io.NORMALIZATION_TYPE_KEY]
min_normalized_value = training_option_dict[
trainval_io.MIN_NORMALIZED_VALUE_KEY]
max_normalized_value = training_option_dict[
trainval_io.MAX_NORMALIZED_VALUE_KEY]
if test_mode:
normalization_file_name = None
else:
normalization_file_name = training_option_dict[
trainval_io.NORMALIZATION_FILE_KEY]
if field_name in RADAR_TO_MINIMUM_DICT:
if normalization_type_string == dl_utils.Z_NORMALIZATION_TYPE_STRING:
min_physical_value = RADAR_TO_MINIMUM_DICT[field_name]
dummy_radar_matrix = numpy.full((1, 1, 1, 1), min_physical_value)
dummy_radar_matrix = dl_utils.normalize_radar_images(
radar_image_matrix=dummy_radar_matrix,
field_names=[field_name],
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value,
test_mode=test_mode,
normalization_table=normalization_table)
min_normalized_value = numpy.ravel(dummy_radar_matrix)[0]
else:
min_normalized_value = numpy.nan
if field_name in RADAR_TO_MAX_DICT:
if normalization_type_string == dl_utils.Z_NORMALIZATION_TYPE_STRING:
max_physical_value = RADAR_TO_MAX_DICT[field_name]
dummy_radar_matrix = numpy.full((1, 1, 1, 1), max_physical_value)
dummy_radar_matrix = dl_utils.normalize_radar_images(
radar_image_matrix=dummy_radar_matrix,
field_names=[field_name],
normalization_type_string=normalization_type_string,
normalization_param_file_name=normalization_file_name,
min_normalized_value=min_normalized_value,
max_normalized_value=max_normalized_value,
test_mode=test_mode,
normalization_table=normalization_table)
max_normalized_value = numpy.ravel(dummy_radar_matrix)[0]
else:
max_normalized_value = numpy.nan
return min_normalized_value, max_normalized_value
def _run(example_file_name, example_indices, num_radar_rows, num_radar_columns,
normalization_file_name, output_dir_name):
"""Plots data augmentation.
This is effectively the main method.
:param example_file_name: See documentation at top of file.
:param example_indices: Same.
:param num_radar_rows: Same.
:param num_radar_columns: Same.
:param normalization_file_name: Same.
:param output_dir_name: Same.
"""
if num_radar_rows <= 0:
num_radar_rows = None
if num_radar_columns <= 0:
num_radar_columns = None
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
print('Reading data from: "{0:s}"...'.format(example_file_name))
example_dict = input_examples.read_example_file(
netcdf_file_name=example_file_name,
read_all_target_vars=True,
include_soundings=False,
num_rows_to_keep=num_radar_rows,
num_columns_to_keep=num_radar_columns,
radar_field_names_to_keep=[RADAR_FIELD_NAME],
radar_heights_to_keep_m_agl=numpy.array([RADAR_HEIGHT_M_AGL],
dtype=int))
if input_examples.REFL_IMAGE_MATRIX_KEY in example_dict:
radar_matrix = example_dict[input_examples.REFL_IMAGE_MATRIX_KEY]
else:
radar_matrix = example_dict[input_examples.RADAR_IMAGE_MATRIX_KEY]
num_examples_total = radar_matrix.shape[0]
error_checking.assert_is_geq_numpy_array(example_indices, 0)
error_checking.assert_is_less_than_numpy_array(example_indices,
num_examples_total)
radar_matrix = radar_matrix[example_indices, ...]
full_storm_id_strings = [
example_dict[input_examples.FULL_IDS_KEY][k] for k in example_indices
]
storm_times_unix_sec = example_dict[
input_examples.STORM_TIMES_KEY][example_indices]
radar_matrix = dl_utils.normalize_radar_images(
radar_image_matrix=radar_matrix,
field_names=[RADAR_FIELD_NAME],
normalization_type_string=NORMALIZATION_TYPE_STRING,
normalization_param_file_name=normalization_file_name)
num_examples = radar_matrix.shape[0]
dummy_target_values = numpy.full(num_examples, 0, dtype=int)
radar_matrix = trainval_io._augment_radar_images(
list_of_predictor_matrices=[radar_matrix],
target_array=dummy_target_values,
x_translations_pixels=X_TRANSLATIONS_PX,
y_translations_pixels=Y_TRANSLATIONS_PX,
ccw_rotation_angles_deg=CCW_ROTATION_ANGLES_DEG,
noise_standard_deviation=NOISE_STANDARD_DEVIATION,
num_noisings=1,
flip_in_x=False,
flip_in_y=False)[0][0]
radar_matrix = dl_utils.denormalize_radar_images(
radar_image_matrix=radar_matrix,
field_names=[RADAR_FIELD_NAME],
normalization_type_string=NORMALIZATION_TYPE_STRING,
normalization_param_file_name=normalization_file_name)
orig_radar_matrix = radar_matrix[:num_examples, ...]
radar_matrix = radar_matrix[num_examples:, ...]
translated_radar_matrix = radar_matrix[:num_examples, ...]
radar_matrix = radar_matrix[num_examples:, ...]
rotated_radar_matrix = radar_matrix[:num_examples, ...]
noised_radar_matrix = radar_matrix[num_examples:, ...]
for i in range(num_examples):
_plot_one_example(orig_radar_matrix=orig_radar_matrix[i, ...],
translated_radar_matrix=translated_radar_matrix[i,
...],
rotated_radar_matrix=rotated_radar_matrix[i, ...],
noised_radar_matrix=noised_radar_matrix[i, ...],
output_dir_name=output_dir_name,
full_storm_id_string=full_storm_id_strings[i],
storm_time_unix_sec=storm_times_unix_sec[i])
