def run_guided_gradcam(orig_model_object,
list_of_input_matrices,
target_layer_name,
class_activation_matrix,
new_model_object=None):
"""Runs guided Grad-CAM.
M = number of rows in grid
N = number of columns in grid
C = number of channels
:param orig_model_object: Original model (trained instance of
`keras.models.Model` or `keras.models.Sequential`).
:param list_of_input_matrices: See doc for `run_gradcam`.
:param target_layer_name: Same.
:param class_activation_matrix: Same.
:param new_model_object: New model (created by `_change_backprop_function`),
to be used for guided backprop.
:return: ggradcam_output_matrix: M-by-N-by-C numpy array of output values.
:return: new_model_object: See input doc.
"""
# Check input args.
error_checking.assert_is_string(target_layer_name)
error_checking.assert_is_list(list_of_input_matrices)
error_checking.assert_is_numpy_array_without_nan(class_activation_matrix)
for q in range(len(list_of_input_matrices)):
error_checking.assert_is_numpy_array(list_of_input_matrices[q])
if list_of_input_matrices[q].shape[0] != 1:
list_of_input_matrices[q] = numpy.expand_dims(
list_of_input_matrices[q], axis=0)
# Do the dirty work.
if new_model_object is None:
_register_guided_backprop()
new_model_object = _change_backprop_function(
model_object=orig_model_object)
input_index = _find_relevant_input_matrix(
list_of_input_matrices=list_of_input_matrices,
num_spatial_dim=len(class_activation_matrix.shape))
saliency_function = _make_saliency_function(model_object=new_model_object,
layer_name=target_layer_name,
input_index=input_index)
saliency_matrix = saliency_function(list_of_input_matrices + [0])[0]
print 'Minimum saliency = {0:.4e} ... max saliency = {1:.4e}'.format(
numpy.min(saliency_matrix), numpy.max(saliency_matrix))
ggradcam_output_matrix = saliency_matrix * class_activation_matrix[
..., numpy.newaxis]
ggradcam_output_matrix = ggradcam_output_matrix[0, ...]
# ggradcam_output_matrix = _normalize_guided_gradcam_output(
# ggradcam_output_matrix[0, ...])
return ggradcam_output_matrix, new_model_object
def _check_region_dict(list_of_cam_matrices, region_dict, pmm_flag):
"""Error-checks dictionary with regions of interest.
:param list_of_cam_matrices: See doc for `_check_in_and_out_matrices`.
:param region_dict: Dictionary with the following keys.
region_dict['list_of_mask_matrices']: Same as `list_of_cam_matrices`, except
that all numpy arrays. Grid cells marked True are in a region of
interest.
region_dict['list_of_polygon_objects']: Triple-nested list of polygons
(instances of `shapely.geometry.Polygon`), demarcating regions of
interest. list_of_polygon_objects[i][j][k] is the [k]th region of
interest for the [j]th input matrix for the [i]th example.
region_dict['percentile_threshold']: Percentile threshold used to create
regions of interest. This is applied separately to each
class-activation matrix for each example.
region_dict['min_class_activation']: Minimum class activation used to create
regions of interest. For a grid cell to be in a region of interest, it
must meet both this and the percentile threshold.
:param pmm_flag: Boolean flag. If True, inputs should contain PMM
(probability-matched mean) composites.
"""
error_checking.assert_is_geq(region_dict[PERCENTILE_THRESHOLD_KEY], 50.)
error_checking.assert_is_less_than(
region_dict[PERCENTILE_THRESHOLD_KEY], 100.
)
error_checking.assert_is_greater(region_dict[MIN_CLASS_ACTIVATION_KEY], 0.)
list_of_mask_matrices = region_dict[MASK_MATRICES_KEY]
list_of_polygon_objects = region_dict[POLYGON_OBJECTS_KEY]
num_input_matrices = len(list_of_cam_matrices)
assert len(list_of_mask_matrices) == num_input_matrices
assert len(list_of_polygon_objects) == num_input_matrices
for j in range(num_input_matrices):
if list_of_cam_matrices[j] is None:
assert list_of_mask_matrices[j] is None
continue
error_checking.assert_is_boolean_numpy_array(list_of_mask_matrices[j])
these_expected_dim = numpy.array(
list_of_cam_matrices[j].shape, dtype=int
)
error_checking.assert_is_numpy_array(
list_of_mask_matrices[j], exact_dimensions=these_expected_dim
)
if pmm_flag:
num_examples = 1
else:
num_examples = list_of_cam_matrices[j].shape[0]
assert len(list_of_polygon_objects[j]) == num_examples
for i in range(num_examples):
error_checking.assert_is_list(list_of_polygon_objects[j][i])
def _check_in_and_out_matrices(predictor_matrices,
num_examples=None,
saliency_matrices=None):
"""Error-checks input and output matrices.
T = number of input tensors to the model
E = number of examples (storm objects)
:param predictor_matrices: length-T list of predictor matrices. Each item
must be a numpy array.
:param num_examples: E in the above discussion. The first axis of each
array must have length E. If you don't know the number of examples,
leave this as None.
:param saliency_matrices: Same as `predictor_matrices` but with saliency
values.
:raises: ValueError: if `predictor_matrices` and `saliency_matrices` have
different lengths.
"""
error_checking.assert_is_list(predictor_matrices)
num_matrices = len(predictor_matrices)
if saliency_matrices is None:
saliency_matrices = [None] * num_matrices
error_checking.assert_is_list(saliency_matrices)
num_saliency_matrices = len(saliency_matrices)
if num_matrices != num_saliency_matrices:
error_string = (
'Number of predictor matrices ({0:d}) should = number of saliency '
'matrices ({1:d}).').format(num_matrices, num_saliency_matrices)
raise ValueError(error_string)
for i in range(num_matrices):
error_checking.assert_is_numpy_array_without_nan(predictor_matrices[i])
if num_examples is not None:
these_expected_dim = numpy.array(
(num_examples, ) + predictor_matrices[i].shape[1:], dtype=int)
error_checking.assert_is_numpy_array(
predictor_matrices[i], exact_dimensions=these_expected_dim)
if saliency_matrices[i] is not None:
error_checking.assert_is_numpy_array_without_nan(
saliency_matrices[i])
these_expected_dim = numpy.array(predictor_matrices[i].shape,
dtype=int)
error_checking.assert_is_numpy_array(
saliency_matrices[i], exact_dimensions=these_expected_dim)
def apply_model_for_each_class(orig_class_probability_matrix, observed_labels,
model_object_by_class):
"""Applies isotonic-regression model for each class.
:param orig_class_probability_matrix: See documentation for
`train_model_for_each_class`.
:param observed_labels: Same.
:param model_object_by_class: Same.
:return: new_class_probability_matrix: Calibrated version of
`orig_class_probability_matrix`.
:raises: ValueError: if number of models != number of columns in
`orig_class_probability_matrix`.
"""
_check_evaluation_pairs(
class_probability_matrix=orig_class_probability_matrix,
observed_labels=observed_labels)
error_checking.assert_is_list(model_object_by_class)
num_classes = orig_class_probability_matrix.shape[1]
if len(model_object_by_class) != num_classes:
error_string = (
'Number of models ({0:d}) should = number of columns in '
'orig_class_probability_matrix ({1:d}).').format(
len(model_object_by_class), num_classes)
raise ValueError(error_string)
num_examples = orig_class_probability_matrix.shape[0]
new_class_probability_matrix = numpy.full((num_examples, num_classes),
numpy.nan)
for k in range(num_classes):
print 'Applying isotonic-regression model for class {0:d}...'.format(k)
new_class_probability_matrix[:, k] = model_object_by_class[k].predict(
orig_class_probability_matrix[:, k])
# Ensure that sum of class probabilities = 1 for each example.
for i in range(num_examples):
new_class_probability_matrix[i, :] = (
new_class_probability_matrix[i, :] /
numpy.sum(new_class_probability_matrix[i, :]))
_check_evaluation_pairs(
class_probability_matrix=new_class_probability_matrix,
observed_labels=observed_labels)
return new_class_probability_matrix
def _polygon_list_to_vertex_list(polygon_objects_grid_coords):
"""Converts list of polygons to list of vertices.
V = total number of vertices
:param polygon_objects_grid_coords: List of polygons created by
`polygons_from_pixel_to_grid_coords`.
:return: vertex_rows: length-V numpy array of row coordinates.
:return: vertex_columns: length-V numpy array of column coordinates.
:return: vertex_to_polygon_indices: length-V numpy array of indices.
If vertex_to_polygon_indices[i] = j, the [i]th vertex comes from the
[j]th input polygon.
"""
error_checking.assert_is_list(polygon_objects_grid_coords)
num_polygons = len(polygon_objects_grid_coords)
if num_polygons > 0:
error_checking.assert_is_numpy_array(numpy.array(
polygon_objects_grid_coords, dtype=object),
num_dimensions=1)
vertex_rows = []
vertex_columns = []
vertex_to_polygon_indices = []
for i in range(num_polygons):
this_num_vertices = len(polygon_objects_grid_coords[i].exterior.xy[1])
vertex_rows += polygon_objects_grid_coords[i].exterior.xy[1]
vertex_columns += polygon_objects_grid_coords[i].exterior.xy[0]
vertex_to_polygon_indices += [i] * this_num_vertices
if i == num_polygons - 1:
continue
vertex_rows += [numpy.nan]
vertex_columns += [numpy.nan]
vertex_to_polygon_indices += [-1]
return (numpy.array(vertex_rows), numpy.array(vertex_columns),
numpy.array(vertex_to_polygon_indices, dtype=int))
def write_pmm_file(pickle_file_name,
list_of_mean_input_matrices,
list_of_mean_optimized_matrices,
mean_initial_activation,
mean_final_activation,
threshold_count_matrix,
model_file_name,
standard_bwo_file_name,
pmm_metadata_dict,
monte_carlo_dict=None):
"""Writes mean backwards-optimized map to Pickle file.
This is a mean over many examples, created by PMM (probability-matched
means).
T = number of input tensors to the model
:param pickle_file_name: Path to output file.
:param list_of_mean_input_matrices: length-T list of numpy arrays, where
list_of_mean_input_matrices[i] is the mean (over many examples) of the
[i]th input tensor to the model. list_of_mean_input_matrices[i] should
have the same dimensions as the [i]th input tensor, except without the
first axis.
:param list_of_mean_optimized_matrices: Same as
`list_of_mean_input_matrices` (and with the same dimensions), but for
optimized learning examples. In other words,
`list_of_mean_input_matrices` contains the mean input and
`list_of_mean_optimized_matrices` contains the mean output.
:param mean_initial_activation: Mean initial activation of relevant model
component (before backwards optimization).
:param mean_fimal_activation: Mean final activation of relevant model
component (after backwards optimization).
:param threshold_count_matrix: See doc for
`prob_matched_means.run_pmm_many_variables`.
:param model_file_name: Path to file with trained model (readable by
`cnn.read_model`).
:param standard_bwo_file_name: Path to file with standard
backwards-optimization output (readable by `read_standard_file`).
:param pmm_metadata_dict: Dictionary created by
`prob_matched_means.check_input_args`.
:param monte_carlo_dict: Dictionary with results of Monte Carlo significance
test. Must contain keys listed in `monte_carlo.check_output`, plus the
following.
monte_carlo_dict['baseline_file_name']: Path to saliency file for baseline
set (readable by `read_standard_file`), against which the trial set
(contained in `standard_bwo_file_name`) was compared.
:raises: ValueError: if `list_of_mean_input_matrices` and
`list_of_mean_optimized_matrices` have different lengths.
"""
error_checking.assert_is_string(model_file_name)
error_checking.assert_is_string(standard_bwo_file_name)
error_checking.assert_is_list(list_of_mean_input_matrices)
error_checking.assert_is_list(list_of_mean_optimized_matrices)
num_input_matrices = len(list_of_mean_input_matrices)
num_output_matrices = len(list_of_mean_optimized_matrices)
if num_input_matrices != num_output_matrices:
error_string = (
'Number of input matrices ({0:d}) should equal number of output '
'matrices ({1:d}).').format(num_input_matrices,
num_output_matrices)
raise ValueError(error_string)
for i in range(num_input_matrices):
error_checking.assert_is_numpy_array_without_nan(
list_of_mean_input_matrices[i])
error_checking.assert_is_numpy_array_without_nan(
list_of_mean_optimized_matrices[i])
these_expected_dim = numpy.array(list_of_mean_input_matrices[i].shape,
dtype=int)
error_checking.assert_is_numpy_array(
list_of_mean_optimized_matrices[i],
exact_dimensions=these_expected_dim)
if threshold_count_matrix is not None:
error_checking.assert_is_integer_numpy_array(threshold_count_matrix)
error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0)
spatial_dimensions = numpy.array(
list_of_mean_input_matrices[0].shape[:-1], dtype=int)
error_checking.assert_is_numpy_array(
threshold_count_matrix, exact_dimensions=spatial_dimensions)
if monte_carlo_dict is not None:
monte_carlo.check_output(monte_carlo_dict)
error_checking.assert_is_string(
monte_carlo_dict[monte_carlo.BASELINE_FILE_KEY])
for i in range(num_input_matrices):
assert numpy.allclose(
list_of_mean_optimized_matrices[i],
monte_carlo_dict[monte_carlo.TRIAL_PMM_MATRICES_KEY][i],
atol=TOLERANCE)
mean_optimization_dict = {
MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices,
MEAN_OPTIMIZED_MATRICES_KEY: list_of_mean_optimized_matrices,
MEAN_INITIAL_ACTIVATION_KEY: mean_initial_activation,
MEAN_FINAL_ACTIVATION_KEY: mean_final_activation,
THRESHOLD_COUNTS_KEY: threshold_count_matrix,
MODEL_FILE_KEY: model_file_name,
STANDARD_FILE_NAME_KEY: standard_bwo_file_name,
PMM_METADATA_KEY: pmm_metadata_dict,
MONTE_CARLO_DICT_KEY: monte_carlo_dict
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(mean_optimization_dict, pickle_file_handle)
pickle_file_handle.close()
def _check_in_and_out_matrices(predictor_matrices,
num_examples=None,
cam_matrices=None,
guided_cam_matrices=None):
"""Error-checks input and output matrices.
T = number of input tensors to the model
E = number of examples (storm objects)
:param predictor_matrices: length-T list of predictor matrices. Each item
must be a numpy array.
:param num_examples: E in the above discussion. The first axis of each
array must have length E. If you don't know the number of examples,
leave this as None.
:param cam_matrices: length-T list of class-activation maps.
cam_matrices[i] is the class-activation map for predictor_matrices[i]
and should have the same dimensions as predictor_matrices[i], except
without the last dimension, which is the channel dimension. If the
[i]th tensor is not connected to the target layer, cam_matrices[i]
should be None.
:param guided_cam_matrices: length-T list of guided class-activation maps.
Same as cam_matrices[i], except that cam_matrices[i] should have the
same dimensions as predictor_matrices[i], including the last dimension.
:raises: ValueError: if `cam_matrices` or `guided_cam_matrices` has
different length than `predictor_matrices`.
"""
error_checking.assert_is_list(predictor_matrices)
num_matrices = len(predictor_matrices)
if cam_matrices is None:
cam_matrices = [None] * num_matrices
if guided_cam_matrices is None:
guided_cam_matrices = [None] * num_matrices
num_cam_matrices = len(cam_matrices)
num_guided_cam_matrices = len(guided_cam_matrices)
if num_matrices != num_cam_matrices:
error_string = (
'Number of predictor matrices ({0:d}) should = number of CAM '
'matrices ({1:d}).').format(num_matrices, num_cam_matrices)
raise ValueError(error_string)
if num_matrices != num_guided_cam_matrices:
error_string = (
'Number of predictor matrices ({0:d}) should = number of guided-CAM'
' matrices ({1:d}).').format(num_matrices, num_guided_cam_matrices)
raise ValueError(error_string)
for i in range(num_matrices):
error_checking.assert_is_numpy_array_without_nan(predictor_matrices[i])
if num_examples is not None:
these_expected_dim = numpy.array(
(num_examples, ) + predictor_matrices[i].shape[1:], dtype=int)
error_checking.assert_is_numpy_array(
predictor_matrices[i], exact_dimensions=these_expected_dim)
if cam_matrices[i] is not None:
error_checking.assert_is_numpy_array_without_nan(cam_matrices[i])
these_expected_dim = numpy.array(predictor_matrices[i].shape[:-1],
dtype=int)
error_checking.assert_is_numpy_array(
cam_matrices[i], exact_dimensions=these_expected_dim)
if guided_cam_matrices[i] is not None:
error_checking.assert_is_numpy_array_without_nan(
guided_cam_matrices[i])
these_expected_dim = numpy.array(predictor_matrices[i].shape,
dtype=int)
error_checking.assert_is_numpy_array(
guided_cam_matrices[i], exact_dimensions=these_expected_dim)
def test_assert_is_list_numpy_array(self):
"""Checks assert_is_list when input is numpy array."""
with self.assertRaises(TypeError):
error_checking.assert_is_list(REAL_NUMPY_ARRAY)
def test_assert_is_list_tuple(self):
"""Checks assert_is_list when input is tuple."""
with self.assertRaises(TypeError):
error_checking.assert_is_list(REAL_NUMBER_TUPLE)
def write_predictions(
pickle_file_name, denorm_recon_radar_matrices, full_storm_id_strings,
storm_times_unix_sec, mse_by_example, upconvnet_file_name):
"""Writes predictions (reconstructed radar images) to Pickle file.
E = number of examples
:param pickle_file_name: Path to output file.
:param denorm_recon_radar_matrices: 1-D list of denormalized, reconstructed
radar images. Each item must be a 4-D or 5-D numpy array where the
first axis has length E.
:param full_storm_id_strings: length-E list of storm IDs.
:param storm_times_unix_sec: length-E numpy array of valid times.
:param mse_by_example: length-E numpy array of mean squared errors (in
normalized, not physical, units).
:param upconvnet_file_name: Path to upconvnet that generated the
reconstructed images (readable by `cnn.read_model`).
"""
error_checking.assert_is_string_list(full_storm_id_strings)
error_checking.assert_is_numpy_array(
numpy.array(full_storm_id_strings), num_dimensions=1
)
num_examples = len(full_storm_id_strings)
error_checking.assert_is_list(denorm_recon_radar_matrices)
for this_matrix in denorm_recon_radar_matrices:
error_checking.assert_is_numpy_array_without_nan(this_matrix)
this_num_dimensions = len(this_matrix.shape)
error_checking.assert_is_geq(this_num_dimensions, 4)
error_checking.assert_is_leq(this_num_dimensions, 5)
these_expected_dim = numpy.array(
(num_examples,) + this_matrix.shape[1:], dtype=int
)
error_checking.assert_is_numpy_array(
this_matrix, exact_dimensions=these_expected_dim)
these_expected_dim = numpy.array([num_examples], dtype=int)
error_checking.assert_is_integer_numpy_array(storm_times_unix_sec)
error_checking.assert_is_numpy_array(
storm_times_unix_sec, exact_dimensions=these_expected_dim)
error_checking.assert_is_geq_numpy_array(mse_by_example, 0.)
error_checking.assert_is_numpy_array(
mse_by_example, exact_dimensions=these_expected_dim)
error_checking.assert_is_string(upconvnet_file_name)
prediction_dict = {
RECON_IMAGE_MATRICES_KEY: denorm_recon_radar_matrices,
MEAN_SQUARED_ERRORS_KEY: mse_by_example,
STORM_TIMES_KEY: storm_times_unix_sec,
FULL_STORM_IDS_KEY: full_storm_id_strings,
UPCONVNET_FILE_KEY: upconvnet_file_name
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(prediction_dict, pickle_file_handle)
pickle_file_handle.close()
def write_standard_file(pickle_file_name,
list_of_input_matrices,
list_of_saliency_matrices,
storm_ids,
storm_times_unix_sec,
model_file_name,
saliency_metadata_dict,
sounding_pressure_matrix_pascals=None):
"""Writes saliency maps (one per example) to Pickle file.
T = number of input tensors to the model
E = number of examples (storm objects)
H = number of height levels per sounding
:param pickle_file_name: Path to output file.
:param list_of_input_matrices: length-T list of numpy arrays, containing
predictors (inputs to the model). The first dimension of each array
must have length E.
:param list_of_saliency_matrices: length-T list of numpy arrays, containing
saliency values. list_of_saliency_matrices[i] must have the same
dimensions as list_of_input_matrices[i].
:param storm_ids: length-E list of storm IDs (strings).
:param storm_times_unix_sec: length-E numpy array of storm times.
:param model_file_name: Path to file with trained model (readable by
`cnn.read_model`).
:param saliency_metadata_dict: Dictionary created by `check_metadata`.
:param sounding_pressure_matrix_pascals: E-by-H numpy array of pressure
levels in soundings. Useful only when the model input contains
soundings with no pressure, because it is needed to plot soundings.
:raises: ValueError: if `list_of_input_matrices` and
`list_of_saliency_matrices` have different lengths.
"""
error_checking.assert_is_string(model_file_name)
error_checking.assert_is_string_list(storm_ids)
error_checking.assert_is_numpy_array(numpy.array(storm_ids),
num_dimensions=1)
num_storm_objects = len(storm_ids)
these_expected_dim = numpy.array([num_storm_objects], dtype=int)
error_checking.assert_is_integer_numpy_array(storm_times_unix_sec)
error_checking.assert_is_numpy_array(storm_times_unix_sec,
exact_dimensions=these_expected_dim)
error_checking.assert_is_list(list_of_input_matrices)
error_checking.assert_is_list(list_of_saliency_matrices)
num_input_matrices = len(list_of_input_matrices)
num_saliency_matrices = len(list_of_saliency_matrices)
if num_input_matrices != num_saliency_matrices:
error_string = (
'Number of input matrices ({0:d}) should equal number of saliency '
'matrices ({1:d}).').format(num_input_matrices,
num_saliency_matrices)
raise ValueError(error_string)
for i in range(num_input_matrices):
error_checking.assert_is_numpy_array_without_nan(
list_of_input_matrices[i])
error_checking.assert_is_numpy_array_without_nan(
list_of_saliency_matrices[i])
these_expected_dim = numpy.array(
(num_storm_objects, ) + list_of_input_matrices[i].shape[1:],
dtype=int)
error_checking.assert_is_numpy_array(
list_of_input_matrices[i], exact_dimensions=these_expected_dim)
these_expected_dim = numpy.array(list_of_input_matrices[i].shape,
dtype=int)
error_checking.assert_is_numpy_array(
list_of_saliency_matrices[i], exact_dimensions=these_expected_dim)
if sounding_pressure_matrix_pascals is not None:
error_checking.assert_is_numpy_array_without_nan(
sounding_pressure_matrix_pascals)
error_checking.assert_is_greater_numpy_array(
sounding_pressure_matrix_pascals, 0.)
error_checking.assert_is_numpy_array(sounding_pressure_matrix_pascals,
num_dimensions=2)
these_expected_dim = numpy.array(
(num_storm_objects, ) + sounding_pressure_matrix_pascals.shape[1:],
dtype=int)
error_checking.assert_is_numpy_array(
sounding_pressure_matrix_pascals,
exact_dimensions=these_expected_dim)
saliency_dict = {
INPUT_MATRICES_KEY: list_of_input_matrices,
SALIENCY_MATRICES_KEY: list_of_saliency_matrices,
STORM_IDS_KEY: storm_ids,
STORM_TIMES_KEY: storm_times_unix_sec,
MODEL_FILE_NAME_KEY: model_file_name,
COMPONENT_TYPE_KEY: saliency_metadata_dict[COMPONENT_TYPE_KEY],
TARGET_CLASS_KEY: saliency_metadata_dict[TARGET_CLASS_KEY],
LAYER_NAME_KEY: saliency_metadata_dict[LAYER_NAME_KEY],
IDEAL_ACTIVATION_KEY: saliency_metadata_dict[IDEAL_ACTIVATION_KEY],
NEURON_INDICES_KEY: saliency_metadata_dict[NEURON_INDICES_KEY],
CHANNEL_INDEX_KEY: saliency_metadata_dict[CHANNEL_INDEX_KEY],
SOUNDING_PRESSURES_KEY: sounding_pressure_matrix_pascals
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(saliency_dict, pickle_file_handle)
pickle_file_handle.close()
def write_pmm_file(pickle_file_name, list_of_mean_input_matrices,
list_of_mean_optimized_matrices, threshold_count_matrix,
standard_bwo_file_name, pmm_metadata_dict):
"""Writes mean backwards-optimized map to Pickle file.
This is a mean over many examples, created by PMM (probability-matched
means).
T = number of input tensors to the model
:param pickle_file_name: Path to output file.
:param list_of_mean_input_matrices: length-T list of numpy arrays, where
list_of_mean_input_matrices[i] is the mean (over many examples) of the
[i]th input tensor to the model. list_of_mean_input_matrices[i] should
have the same dimensions as the [i]th input tensor, except without the
first axis.
:param list_of_mean_optimized_matrices: Same as
`list_of_mean_input_matrices` (and with the same dimensions), but for
optimized learning examples. In other words,
`list_of_mean_input_matrices` contains the mean input and
`list_of_mean_optimized_matrices` contains the mean output.
:param threshold_count_matrix: See doc for
`prob_matched_means.run_pmm_many_variables`.
:param standard_bwo_file_name: Path to file with standard
backwards-optimization output (readable by `read_standard_file`).
:param pmm_metadata_dict: Dictionary created by
`prob_matched_means.check_input_args`.
:raises: ValueError: if `list_of_mean_input_matrices` and
`list_of_mean_optimized_matrices` have different lengths.
"""
error_checking.assert_is_string(standard_bwo_file_name)
error_checking.assert_is_list(list_of_mean_input_matrices)
error_checking.assert_is_list(list_of_mean_optimized_matrices)
num_input_matrices = len(list_of_mean_input_matrices)
num_output_matrices = len(list_of_mean_optimized_matrices)
if num_input_matrices != num_output_matrices:
error_string = (
'Number of input matrices ({0:d}) should equal number of output '
'matrices ({1:d}).').format(num_input_matrices,
num_output_matrices)
raise ValueError(error_string)
for i in range(num_input_matrices):
error_checking.assert_is_numpy_array_without_nan(
list_of_mean_input_matrices[i])
error_checking.assert_is_numpy_array_without_nan(
list_of_mean_optimized_matrices[i])
these_expected_dim = numpy.array(list_of_mean_input_matrices[i].shape,
dtype=int)
error_checking.assert_is_numpy_array(
list_of_mean_optimized_matrices[i],
exact_dimensions=these_expected_dim)
if threshold_count_matrix is not None:
error_checking.assert_is_integer_numpy_array(threshold_count_matrix)
error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0)
spatial_dimensions = numpy.array(
list_of_mean_input_matrices[0].shape[:-1], dtype=int)
error_checking.assert_is_numpy_array(
threshold_count_matrix, exact_dimensions=spatial_dimensions)
mean_optimization_dict = {
MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices,
MEAN_OPTIMIZED_MATRICES_KEY: list_of_mean_optimized_matrices,
THRESHOLD_COUNTS_KEY: threshold_count_matrix,
STANDARD_FILE_NAME_KEY: standard_bwo_file_name,
PMM_METADATA_KEY: pmm_metadata_dict
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(mean_optimization_dict, pickle_file_handle)
pickle_file_handle.close()
def plot_many_soundings(list_of_metpy_dictionaries,
title_strings,
num_panel_rows,
output_file_name,
temp_directory_name=None,
option_dict=None):
"""Creates paneled figure with many soundings.
N = number of soundings to plot
:param list_of_metpy_dictionaries: length-N list of dictionaries. Each
dictionary must satisfy the input format for `sounding_dict_for_metpy`
in `plot_sounding`.
:param title_strings: length-N list of titles.
:param num_panel_rows: Number of rows in paneled figure.
:param output_file_name: Path to output (image) file.
:param temp_directory_name: Name of temporary directory. Each panel will be
stored here, then deleted after the panels have been concatenated into
the final image. If `temp_directory_name is None`, will use the default
temp directory on the local machine.
:param option_dict: See doc for `plot_sounding`.
"""
error_checking.assert_is_numpy_array(numpy.array(title_strings),
num_dimensions=1)
num_soundings = len(title_strings)
error_checking.assert_is_list(list_of_metpy_dictionaries)
error_checking.assert_is_geq(len(list_of_metpy_dictionaries),
num_soundings)
error_checking.assert_is_leq(len(list_of_metpy_dictionaries),
num_soundings)
error_checking.assert_is_integer(num_panel_rows)
error_checking.assert_is_geq(num_panel_rows, 1)
error_checking.assert_is_leq(num_panel_rows, num_soundings)
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
if temp_directory_name is not None:
file_system_utils.mkdir_recursive_if_necessary(
directory_name=temp_directory_name)
temp_file_names = [None] * num_soundings
num_panel_columns = int(numpy.ceil(float(num_soundings) / num_panel_rows))
for i in range(num_panel_rows):
for j in range(num_panel_columns):
this_sounding_index = i * num_panel_columns + j
if this_sounding_index >= num_soundings:
break
plot_sounding(sounding_dict_for_metpy=list_of_metpy_dictionaries[
this_sounding_index],
title_string=title_strings[this_sounding_index],
option_dict=option_dict)
temp_file_names[this_sounding_index] = '{0:s}.jpg'.format(
tempfile.NamedTemporaryFile(dir=temp_directory_name,
delete=False).name)
print('Saving sounding to: "{0:s}"...'.format(
temp_file_names[this_sounding_index]))
pyplot.savefig(temp_file_names[this_sounding_index],
dpi=DOTS_PER_INCH)
pyplot.close()
imagemagick_utils.trim_whitespace(
input_file_name=temp_file_names[this_sounding_index],
output_file_name=temp_file_names[this_sounding_index],
border_width_pixels=SINGLE_IMAGE_BORDER_WIDTH_PX)
imagemagick_utils.resize_image(
input_file_name=temp_file_names[this_sounding_index],
output_file_name=temp_file_names[this_sounding_index],
output_size_pixels=SINGLE_IMAGE_SIZE_PX)
print('Concatenating panels into one figure: "{0:s}"...'.format(
output_file_name))
imagemagick_utils.concatenate_images(
input_file_names=temp_file_names,
output_file_name=output_file_name,
num_panel_rows=num_panel_rows,
num_panel_columns=num_panel_columns,
border_width_pixels=PANELED_IMAGE_BORDER_WIDTH_PX)
for i in range(num_soundings):
os.remove(temp_file_names[i])
def plot_many_2d_grids_without_coords(field_matrix,
field_name_by_panel,
num_panel_rows=None,
figure_object=None,
axes_object_matrix=None,
panel_names=None,
colour_map_object_by_panel=None,
colour_norm_object_by_panel=None,
plot_colour_bar_by_panel=None,
font_size=DEFAULT_FONT_SIZE,
row_major=True):
"""Plots 2-D colour map in each panel (one per field/height pair).
M = number of rows in spatial grid
N = number of columns in spatial grid
P = number of panels (field/height pairs)
This method uses the default colour scheme for each radar field.
If `num_panel_rows is None`, this method needs arguments `figure_object` and
`axes_object_matrix` -- and vice-versa.
:param field_matrix: M-by-N-by-P numpy array of radar values.
:param field_name_by_panel: length-P list of field names.
:param num_panel_rows: Number of rows in paneled figure (different than M,
which is number of rows in spatial grid).
:param figure_object: See doc for `plotting_utils.create_paneled_figure`.
:param axes_object_matrix: See above.
:param panel_names: length-P list of panel names (will be printed at bottoms
of panels). If you do not want panel names, make this None.
:param colour_map_object_by_panel: length-P list of `matplotlib.pyplot.cm`
objects. If this is None, the default will be used for each field.
:param colour_norm_object_by_panel: length-P list of
`matplotlib.colors.BoundaryNorm` objects. If this is None, the default
will be used for each field.
:param plot_colour_bar_by_panel: length-P numpy array of Boolean flags. If
plot_colour_bar_by_panel[k] = True, horizontal colour bar will be
plotted under [k]th panel. If you want to plot colour bar for every
panel, leave this as None.
:param font_size: Font size.
:param row_major: Boolean flag. If True, panels will be filled along rows
first, then down columns. If False, down columns first, then along
rows.
:return: figure_object: See doc for `plotting_utils.create_paneled_figure`.
:return: axes_object_matrix: Same.
:raises: ValueError: if `colour_map_object_by_panel` or
`colour_norm_object_by_panel` has different length than number of
panels.
"""
error_checking.assert_is_boolean(row_major)
error_checking.assert_is_numpy_array(field_matrix, num_dimensions=3)
num_panels = field_matrix.shape[2]
if panel_names is None:
panel_names = [None] * num_panels
if plot_colour_bar_by_panel is None:
plot_colour_bar_by_panel = numpy.full(num_panels, True, dtype=bool)
these_expected_dim = numpy.array([num_panels], dtype=int)
error_checking.assert_is_numpy_array(numpy.array(panel_names),
exact_dimensions=these_expected_dim)
error_checking.assert_is_numpy_array(numpy.array(field_name_by_panel),
exact_dimensions=these_expected_dim)
error_checking.assert_is_boolean_numpy_array(plot_colour_bar_by_panel)
error_checking.assert_is_numpy_array(plot_colour_bar_by_panel,
exact_dimensions=these_expected_dim)
if (colour_map_object_by_panel is None
or colour_norm_object_by_panel is None):
colour_map_object_by_panel = [None] * num_panels
colour_norm_object_by_panel = [None] * num_panels
error_checking.assert_is_list(colour_map_object_by_panel)
error_checking.assert_is_list(colour_norm_object_by_panel)
if len(colour_map_object_by_panel) != num_panels:
error_string = (
'Number of colour maps ({0:d}) should equal number of panels '
'({1:d}).').format(len(colour_map_object_by_panel), num_panels)
raise ValueError(error_string)
if len(colour_norm_object_by_panel) != num_panels:
error_string = (
'Number of colour-normalizers ({0:d}) should equal number of panels'
' ({1:d}).').format(len(colour_norm_object_by_panel), num_panels)
raise ValueError(error_string)
if figure_object is None:
error_checking.assert_is_integer(num_panel_rows)
error_checking.assert_is_geq(num_panel_rows, 1)
error_checking.assert_is_leq(num_panel_rows, num_panels)
num_panel_columns = int(numpy.ceil(float(num_panels) / num_panel_rows))
figure_object, axes_object_matrix = (
plotting_utils.create_paneled_figure(num_rows=num_panel_rows,
num_columns=num_panel_columns,
shared_x_axis=False,
shared_y_axis=False,
keep_aspect_ratio=True))
else:
error_checking.assert_is_numpy_array(axes_object_matrix,
num_dimensions=2)
num_panel_rows = axes_object_matrix.shape[0]
num_panel_columns = axes_object_matrix.shape[1]
if row_major:
order_string = 'C'
else:
order_string = 'F'
for k in range(num_panels):
this_panel_row, this_panel_column = numpy.unravel_index(
k, (num_panel_rows, num_panel_columns), order=order_string)
# this_colour_map_object, this_colour_norm_object = (
# plot_2d_grid_without_coords(
# field_matrix=field_matrix[..., k],
# field_name=field_name_by_panel[k],
# axes_object=axes_object_matrix[
# this_panel_row, this_panel_column],
# annotation_string=panel_names[k], font_size=font_size,
# colour_map_object=colour_map_object_by_panel[k],
# colour_norm_object=colour_norm_object_by_panel[k]
# )
# )
this_colour_map_object, this_colour_norm_object = (
plot_2d_grid_without_coords(
field_matrix=field_matrix[..., k],
field_name=field_name_by_panel[k],
axes_object=axes_object_matrix[this_panel_row,
this_panel_column],
annotation_string=None,
font_size=font_size,
colour_map_object=colour_map_object_by_panel[k],
colour_norm_object=colour_norm_object_by_panel[k]))
if not plot_colour_bar_by_panel[k]:
continue
this_extend_min_flag = field_name_by_panel[k] in SHEAR_VORT_DIV_NAMES
this_colour_bar_object = plotting_utils.plot_colour_bar(
axes_object_or_matrix=axes_object_matrix[this_panel_row,
this_panel_column],
data_matrix=field_matrix[..., k],
colour_map_object=this_colour_map_object,
colour_norm_object=this_colour_norm_object,
orientation_string='horizontal',
extend_min=this_extend_min_flag,
extend_max=True,
fraction_of_axis_length=0.75,
font_size=font_size)
this_colour_bar_object.set_label(panel_names[k].replace('\n', '; '),
fontsize=font_size,
fontweight='bold')
for k in range(num_panel_rows * num_panel_columns):
if k < num_panels:
continue
this_panel_row, this_panel_column = numpy.unravel_index(
k, (num_panel_rows, num_panel_columns), order=order_string)
axes_object_matrix[this_panel_row, this_panel_column].axis('off')
return figure_object, axes_object_matrix
def _check_in_and_out_matrices(
list_of_input_matrices, num_examples, list_of_cam_matrices=None,
list_of_guided_cam_matrices=None):
"""Error-checks input and output matrices for Grad-CAM.
T = number of input tensors to the model
:param list_of_input_matrices: length-T list of numpy arrays, containing
only one example (storm object). list_of_input_matrices[i] must have
the same dimensions as the [i]th input tensor to the model.
:param num_examples: Number of examples expected. If num_examples <= 0,
this method will allow the example dimension to be missing from the
matrices.
:param list_of_cam_matrices: length-T list of numpy arrays.
list_of_cam_matrices[i] will be the class-activation map for
list_of_input_matrices[i]. It will have the same dimensions as
list_of_input_matrices[i], just without the channel dimension. If the
target layer is not connected to the [i]th input tensor,
list_of_cam_matrices[i] will be None.
:param list_of_guided_cam_matrices: length-T list of numpy arrays.
list_of_guided_cam_matrices[i] will be the guided class-activation map
for list_of_input_matrices[i] and will have the same dimensions as
list_of_input_matrices[i]. If the target layer is not connected to the
[i]th input tensor, list_of_guided_cam_matrices[i] will be None.
:return: list_of_input_matrices: Same as input but guaranteed to have
example dimensions.
"""
error_checking.assert_is_list(list_of_input_matrices)
num_input_matrices = len(list_of_input_matrices)
if list_of_cam_matrices is None:
list_of_cam_matrices = [None] * num_input_matrices
if list_of_guided_cam_matrices is None:
list_of_guided_cam_matrices = [None] * num_input_matrices
assert len(list_of_cam_matrices) == num_input_matrices
assert len(list_of_guided_cam_matrices) == num_input_matrices
for i in range(num_input_matrices):
error_checking.assert_is_numpy_array_without_nan(
list_of_input_matrices[i]
)
if num_examples == 1 and list_of_input_matrices[i].shape[0] != 1:
list_of_input_matrices[i] = numpy.expand_dims(
list_of_input_matrices[i], axis=0
)
if num_examples > 0:
assert list_of_input_matrices[i].shape[0] == num_examples
if list_of_cam_matrices[i] is not None:
error_checking.assert_is_numpy_array_without_nan(
list_of_cam_matrices[i]
)
these_expected_dim = numpy.array(
list_of_input_matrices[i].shape[:-1], dtype=int
)
error_checking.assert_is_numpy_array(
list_of_cam_matrices[i], exact_dimensions=these_expected_dim
)
if list_of_guided_cam_matrices[i] is not None:
error_checking.assert_is_numpy_array_without_nan(
list_of_guided_cam_matrices[i]
)
these_expected_dim = numpy.array(
list_of_input_matrices[i].shape, dtype=int
)
error_checking.assert_is_numpy_array(
list_of_guided_cam_matrices[i],
exact_dimensions=these_expected_dim
)
def polygons_from_pixel_to_grid_coords(polygon_objects_pixel_coords,
num_grid_rows, num_grid_columns,
num_pixel_rows, num_pixel_columns,
num_panel_rows, num_panel_columns):
"""Converts polygons from pixel coordinates to grid coordinates.
The input args `num_grid_rows` and `num_grid_columns` are the number of rows
and columns in the data grid. These are *not* the same as the number of
pixel rows and columns in the image.
This method assumes that the image contains one or more panels with gridded
data. Each panel may contain a different variable, but they must all
contain the same grid, with the same aspect ratio and no whitespace border
(between the panels or around the outside of the image).
N = number of polygons
:param polygon_objects_pixel_coords: length-N list of polygons (instances of
`shapely.geometry.Polygon`) with vertices in pixel coordinates, where
the top-left corner is x = y = 0.
:param num_grid_rows: Number of rows in grid.
:param num_grid_columns: Number of columns in grid.
:param num_pixel_rows: Number of pixel rows in image.
:param num_pixel_columns: Number of pixel columns in image.
:param num_panel_rows: Number of panel rows in image.
:param num_panel_columns: Number of panel columns in image.
:return: polygon_objects_grid_coords: length-N list of polygons
(instances of `shapely.geometry.Polygon`) with vertices in grid
coordinates, where the bottom-left corner is x = y = 0.
:return: panel_row_by_polygon: length-N numpy array of panel rows. If
panel_rows[k] = i, the [k]th polygon corresponds to a grid in the [i]th
panel row, where the top row is the 0th.
:return: panel_column_by_polygon: length-N numpy array of panel columns. If
panel_columns[k] = j, the [k]th polygon corresponds to a grid in the
[j]th panel column, where the left column is the 0th.
:raises: ValueError: if one polygon is in multiple panels.
"""
error_checking.assert_is_list(polygon_objects_pixel_coords)
num_polygons = len(polygon_objects_pixel_coords)
if num_polygons == 0:
empty_array = numpy.array([], dtype=int)
return polygon_objects_pixel_coords, empty_array, empty_array
error_checking.assert_is_numpy_array(numpy.array(
polygon_objects_pixel_coords, dtype=object),
num_dimensions=1)
polygon_objects_grid_coords = [None] * num_polygons
panel_row_by_polygon = [None] * num_polygons
panel_column_by_polygon = [None] * num_polygons
for k in range(num_polygons):
these_grid_columns, these_panel_columns = pixel_columns_to_grid_columns(
pixel_column_by_vertex=numpy.array(
polygon_objects_pixel_coords[k].exterior.xy[0]),
num_pixel_columns=num_pixel_columns,
num_panel_columns=num_panel_columns,
num_grid_columns=num_grid_columns,
assert_same_panel=True)
panel_column_by_polygon[k] = these_panel_columns[0]
these_grid_rows, these_panel_rows = pixel_rows_to_grid_rows(
pixel_row_by_vertex=numpy.array(
polygon_objects_pixel_coords[k].exterior.xy[1]),
num_pixel_rows=num_pixel_rows,
num_panel_rows=num_panel_rows,
num_grid_rows=num_grid_rows,
assert_same_panel=True)
panel_row_by_polygon[k] = these_panel_rows[0]
polygon_objects_grid_coords[k] = (
polygons.vertex_arrays_to_polygon_object(
exterior_x_coords=these_grid_columns,
exterior_y_coords=these_grid_rows))
panel_row_by_polygon = numpy.array(panel_row_by_polygon, dtype=int)
panel_column_by_polygon = numpy.array(panel_column_by_polygon, dtype=int)
return (polygon_objects_grid_coords, panel_row_by_polygon,
panel_column_by_polygon)
def write_standard_file(pickle_file_name,
list_of_input_matrices,
class_activation_matrix,
ggradcam_output_matrix,
model_file_name,
storm_ids,
storm_times_unix_sec,
target_class,
target_layer_name,
sounding_pressure_matrix_pascals=None):
"""Writes class-activation maps (one for each example) to Pickle file.
T = number of input tensors to the model
E = number of examples (storm objects)
H = number of height levels per sounding
:param pickle_file_name: Path to output file.
:param list_of_input_matrices: length-T list of numpy arrays, containing
denormalized input data for all examples. The first axis of each array
must have length E.
:param class_activation_matrix: numpy array, where
class_activation_matrix[i, ...] is the activation matrix for the [i]th
example, created by `run_gradcam`.
:param ggradcam_output_matrix: numpy array, where
ggradcam_output_matrix[i, ...] contains output of guided Grad-CAM for
the [i]th example, created by `run_guided_gradcam`.
:param model_file_name: Path to file with trained CNN (readable by
`cnn.read_model`).
:param storm_ids: length-E list of storm IDs (strings).
:param storm_times_unix_sec: length-E numpy array of storm times.
:param target_class: See doc for `run_gradcam`.
:param target_layer_name: Same.
:param sounding_pressure_matrix_pascals: E-by-H numpy array of pressure
levels in soundings. Useful when model input contains soundings with no
pressure variable, since pressure is needed to plot soundings.
:raises: ValueError: if `list_of_input_matrices` and
`class_activation_matrix` have non-matching dimensions.
"""
error_checking.assert_is_string(model_file_name)
error_checking.assert_is_integer(target_class)
error_checking.assert_is_geq(target_class, 0)
error_checking.assert_is_string(target_layer_name)
error_checking.assert_is_string_list(storm_ids)
error_checking.assert_is_numpy_array(numpy.array(storm_ids),
num_dimensions=1)
num_storm_objects = len(storm_ids)
error_checking.assert_is_integer_numpy_array(storm_times_unix_sec)
error_checking.assert_is_numpy_array(storm_times_unix_sec,
exact_dimensions=numpy.array(
[num_storm_objects]))
error_checking.assert_is_numpy_array_without_nan(class_activation_matrix)
error_checking.assert_is_geq(len(class_activation_matrix.shape), 2)
these_expected_dim = numpy.array(
(num_storm_objects, ) + class_activation_matrix.shape[1:], dtype=int)
error_checking.assert_is_numpy_array(class_activation_matrix,
exact_dimensions=these_expected_dim)
error_checking.assert_is_numpy_array_without_nan(ggradcam_output_matrix)
num_channels = ggradcam_output_matrix.shape[-1]
these_expected_dim = numpy.array(class_activation_matrix.shape +
(num_channels, ),
dtype=int)
error_checking.assert_is_numpy_array(ggradcam_output_matrix,
exact_dimensions=these_expected_dim)
error_checking.assert_is_list(list_of_input_matrices)
for this_input_matrix in list_of_input_matrices:
error_checking.assert_is_numpy_array_without_nan(this_input_matrix)
these_expected_dim = numpy.array(
(num_storm_objects, ) + this_input_matrix.shape[1:], dtype=int)
error_checking.assert_is_numpy_array(
this_input_matrix, exact_dimensions=these_expected_dim)
if sounding_pressure_matrix_pascals is not None:
error_checking.assert_is_numpy_array(sounding_pressure_matrix_pascals,
num_dimensions=2)
these_expected_dim = numpy.array(
(num_storm_objects, ) + sounding_pressure_matrix_pascals.shape[1:],
dtype=int)
error_checking.assert_is_numpy_array(
sounding_pressure_matrix_pascals,
exact_dimensions=these_expected_dim)
gradcam_dict = {
INPUT_MATRICES_KEY: list_of_input_matrices,
CLASS_ACTIVATIONS_KEY: class_activation_matrix,
GUIDED_GRADCAM_KEY: ggradcam_output_matrix,
MODEL_FILE_NAME_KEY: model_file_name,
STORM_IDS_KEY: storm_ids,
STORM_TIMES_KEY: storm_times_unix_sec,
TARGET_CLASS_KEY: target_class,
TARGET_LAYER_KEY: target_layer_name,
SOUNDING_PRESSURES_KEY: sounding_pressure_matrix_pascals
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(gradcam_dict, pickle_file_handle)
pickle_file_handle.close()
def write_pmm_file(pickle_file_name, list_of_mean_input_matrices,
mean_class_activation_matrix, mean_ggradcam_output_matrix,
threshold_count_matrix, standard_gradcam_file_name,
pmm_metadata_dict):
"""Writes mean class-activation map to Pickle file.
This is a mean over many examples, created by PMM (probability-matched
means).
:param pickle_file_name: Path to output file.
:param list_of_mean_input_matrices: Same as input `list_of_input_matrices`
to `write_standard_file`, but without the first axis.
:param mean_class_activation_matrix: Same as input `class_activation_matrix`
to `write_standard_file`, but without the first axis.
:param mean_ggradcam_output_matrix: Same as input `ggradcam_output_matrix`
to `write_standard_file`, but without the first axis.
:param threshold_count_matrix: See doc for
`prob_matched_means.run_pmm_many_variables`.
:param standard_gradcam_file_name: Path to file with standard Grad-CAM
output (readable by `read_standard_file`).
:param pmm_metadata_dict: Dictionary created by
`prob_matched_means.check_input_args`.
"""
# TODO(thunderhoser): This method currently does not deal with sounding
# pressures.
error_checking.assert_is_string(standard_gradcam_file_name)
error_checking.assert_is_numpy_array_without_nan(
mean_class_activation_matrix)
spatial_dimensions = numpy.array(mean_class_activation_matrix.shape,
dtype=int)
error_checking.assert_is_numpy_array_without_nan(
mean_ggradcam_output_matrix)
num_channels = mean_ggradcam_output_matrix.shape[-1]
these_expected_dim = numpy.array(mean_class_activation_matrix.shape +
(num_channels, ),
dtype=int)
error_checking.assert_is_numpy_array(mean_ggradcam_output_matrix,
exact_dimensions=these_expected_dim)
if threshold_count_matrix is not None:
error_checking.assert_is_integer_numpy_array(threshold_count_matrix)
error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0)
error_checking.assert_is_numpy_array(
threshold_count_matrix, exact_dimensions=spatial_dimensions)
error_checking.assert_is_list(list_of_mean_input_matrices)
for this_input_matrix in list_of_mean_input_matrices:
error_checking.assert_is_numpy_array_without_nan(this_input_matrix)
mean_gradcam_dict = {
MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices,
MEAN_CLASS_ACTIVATIONS_KEY: mean_class_activation_matrix,
MEAN_GUIDED_GRADCAM_KEY: mean_ggradcam_output_matrix,
THRESHOLD_COUNTS_KEY: threshold_count_matrix,
STANDARD_FILE_NAME_KEY: standard_gradcam_file_name,
PMM_METADATA_KEY: pmm_metadata_dict
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(mean_gradcam_dict, pickle_file_handle)
pickle_file_handle.close()
def run_gradcam(model_object, list_of_input_matrices, target_class,
target_layer_name):
"""Runs Grad-CAM.
T = number of input tensors to the model
:param model_object: Trained instance of `keras.models.Model` or
`keras.models.Sequential`.
:param list_of_input_matrices: length-T list of numpy arrays, containing
only one example (storm object). list_of_input_matrices[i] must have
the same dimensions as the [i]th input tensor to the model.
:param target_class: Activation maps will be created for this class. Must
be an integer in 0...(K - 1), where K = number of classes.
:param target_layer_name: Name of target layer. Neuron-importance weights
will be based on activations in this layer.
:return: class_activation_matrix: Class-activation matrix. Dimensions of
this numpy array will be the spatial dimensions of whichever input
tensor feeds into the target layer. For example, if the given input
tensor is 2-dimensional with M rows and N columns, this array will be
M x N.
"""
# Check input args.
error_checking.assert_is_string(target_layer_name)
error_checking.assert_is_list(list_of_input_matrices)
for q in range(len(list_of_input_matrices)):
error_checking.assert_is_numpy_array(list_of_input_matrices[q])
if list_of_input_matrices[q].shape[0] != 1:
list_of_input_matrices[q] = numpy.expand_dims(
list_of_input_matrices[q], axis=0)
# Create loss tensor.
output_layer_object = model_object.layers[-1].output
num_output_neurons = output_layer_object.get_shape().as_list()[-1]
if num_output_neurons == 1:
error_checking.assert_is_leq(target_class, 1)
print model_object.layers[-1].input
if target_class == 1:
# loss_tensor = model_object.layers[-1].output[..., 0]
loss_tensor = model_object.layers[-1].input[..., 0]
else:
# loss_tensor = -1 * model_object.layers[-1].output[..., 0]
loss_tensor = -1 * model_object.layers[-1].input[..., 0]
else:
error_checking.assert_is_less_than(target_class, num_output_neurons)
# loss_tensor = model_object.layers[-1].output[..., target_class]
loss_tensor = model_object.layers[-1].input[..., target_class]
# Create gradient function.
target_layer_activation_tensor = model_object.get_layer(
name=target_layer_name).output
gradient_tensor = _compute_gradients(loss_tensor,
[target_layer_activation_tensor])[0]
gradient_tensor = _normalize_tensor(gradient_tensor)
if isinstance(model_object.input, list):
list_of_input_tensors = model_object.input
else:
list_of_input_tensors = [model_object.input]
gradient_function = K.function(
list_of_input_tensors,
[target_layer_activation_tensor, gradient_tensor])
# Evaluate gradient function.
target_layer_activation_matrix, gradient_matrix = gradient_function(
list_of_input_matrices)
target_layer_activation_matrix = target_layer_activation_matrix[0, ...]
gradient_matrix = gradient_matrix[0, ...]
# Compute class-activation matrix.
mean_weight_by_filter = numpy.mean(gradient_matrix, axis=(0, 1))
class_activation_matrix = numpy.ones(
target_layer_activation_matrix.shape[:-1])
num_filters = len(mean_weight_by_filter)
for m in range(num_filters):
class_activation_matrix += (mean_weight_by_filter[m] *
target_layer_activation_matrix[..., m])
input_index = _find_relevant_input_matrix(
list_of_input_matrices=list_of_input_matrices,
num_spatial_dim=len(class_activation_matrix.shape))
spatial_dimensions = numpy.array(
list_of_input_matrices[input_index].shape[1:-1], dtype=int)
class_activation_matrix = _upsample_cam(
class_activation_matrix=class_activation_matrix,
new_dimensions=spatial_dimensions)
class_activation_matrix[class_activation_matrix < 0.] = 0.
# denominator = numpy.maximum(numpy.max(class_activation_matrix), K.epsilon())
# return class_activation_matrix / denominator
return class_activation_matrix
def write_pmm_file(
pickle_file_name, list_of_mean_input_matrices,
list_of_mean_saliency_matrices, threshold_count_matrix, model_file_name,
standard_saliency_file_name, pmm_metadata_dict, monte_carlo_dict=None):
"""Writes mean saliency map to Pickle file.
This is a mean over many examples, created by PMM (probability-matched
means).
:param pickle_file_name: Path to output file.
:param list_of_mean_input_matrices: Same as input `list_of_input_matrices`
to `write_standard_file`, but without the first axis.
:param list_of_mean_saliency_matrices: Same as input
`list_of_saliency_matrices` to `write_standard_file`, but without the
first axis.
:param threshold_count_matrix: See doc for
`prob_matched_means.run_pmm_many_variables`.
:param model_file_name: Path to file with trained model (readable by
`cnn.read_model`).
:param standard_saliency_file_name: Path to file with standard saliency
output (readable by `read_standard_file`).
:param pmm_metadata_dict: Dictionary created by
`prob_matched_means.check_input_args`.
:param monte_carlo_dict: Dictionary with results of Monte Carlo significance
test. Must contain keys listed in `monte_carlo.check_output`, plus the
following.
monte_carlo_dict['baseline_file_name']: Path to saliency file for baseline
set (readable by `read_standard_file`), against which the trial set
(contained in `standard_saliency_file_name`) was compared.
:raises: ValueError: if `list_of_mean_input_matrices` and
`list_of_mean_saliency_matrices` have different lengths.
"""
# TODO(thunderhoser): This method currently does not deal with sounding
# pressures.
error_checking.assert_is_string(model_file_name)
error_checking.assert_is_string(standard_saliency_file_name)
error_checking.assert_is_list(list_of_mean_input_matrices)
error_checking.assert_is_list(list_of_mean_saliency_matrices)
num_input_matrices = len(list_of_mean_input_matrices)
num_saliency_matrices = len(list_of_mean_saliency_matrices)
if num_input_matrices != num_saliency_matrices:
error_string = (
'Number of input matrices ({0:d}) should equal number of saliency '
'matrices ({1:d}).'
).format(num_input_matrices, num_saliency_matrices)
raise ValueError(error_string)
for i in range(num_input_matrices):
error_checking.assert_is_numpy_array_without_nan(
list_of_mean_input_matrices[i])
error_checking.assert_is_numpy_array_without_nan(
list_of_mean_saliency_matrices[i])
these_expected_dim = numpy.array(
list_of_mean_input_matrices[i].shape, dtype=int)
error_checking.assert_is_numpy_array(
list_of_mean_saliency_matrices[i],
exact_dimensions=these_expected_dim)
if threshold_count_matrix is not None:
error_checking.assert_is_integer_numpy_array(threshold_count_matrix)
error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0)
spatial_dimensions = numpy.array(
list_of_mean_input_matrices[0].shape[:-1], dtype=int)
error_checking.assert_is_numpy_array(
threshold_count_matrix, exact_dimensions=spatial_dimensions)
if monte_carlo_dict is not None:
monte_carlo.check_output(monte_carlo_dict)
error_checking.assert_is_string(
monte_carlo_dict[monte_carlo.BASELINE_FILE_KEY]
)
for i in range(num_input_matrices):
assert numpy.allclose(
list_of_mean_saliency_matrices[i],
monte_carlo_dict[monte_carlo.TRIAL_PMM_MATRICES_KEY][i],
atol=TOLERANCE
)
mean_saliency_dict = {
MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices,
MEAN_SALIENCY_MATRICES_KEY: list_of_mean_saliency_matrices,
THRESHOLD_COUNTS_KEY: threshold_count_matrix,
MODEL_FILE_KEY: model_file_name,
STANDARD_FILE_NAME_KEY: standard_saliency_file_name,
PMM_METADATA_KEY: pmm_metadata_dict,
MONTE_CARLO_DICT_KEY: monte_carlo_dict
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(mean_saliency_dict, pickle_file_handle)
pickle_file_handle.close()
def plot_many_2d_grids_without_coords(
field_matrix, field_name_by_panel, num_panel_rows, panel_names=None,
colour_map_object_by_panel=None, colour_norm_object_by_panel=None,
plot_colour_bar_by_panel=None,
figure_width_inches=DEFAULT_FIGURE_WIDTH_INCHES,
figure_height_inches=DEFAULT_FIGURE_HEIGHT_INCHES,
font_size=DEFAULT_FONT_SIZE, row_major=True):
"""Plots 2-D colour map in each panel (one per field/height pair).
M = number of rows in spatial grid
N = number of columns in spatial grid
P = number of panels (field/height pairs)
This method uses the default colour scheme for each radar field.
:param field_matrix: M-by-N-by-P numpy array of radar values.
:param field_name_by_panel: length-P list of field names.
:param num_panel_rows: Number of rows in paneled figure (different than M,
which is number of rows in spatial grid).
:param panel_names: length-P list of panel names (will be printed at bottoms
of panels). If you do not want panel names, make this None.
:param colour_map_object_by_panel: length-P list of `matplotlib.pyplot.cm`
objects. If this is None, the default will be used for each field.
:param colour_norm_object_by_panel: length-P list of
`matplotlib.colors.BoundaryNorm` objects. If this is None, the default
will be used for each field.
:param plot_colour_bar_by_panel: length-P numpy array of Boolean flags. If
plot_colour_bar_by_panel[k] = True, horizontal colour bar will be
plotted under [k]th panel. If you want to plot colour bar for every
panel, leave this as None.
:param figure_width_inches: Figure width.
:param figure_height_inches: Figure height.
:param font_size: Font size.
:param row_major: Boolean flag. If True, panels will be filled along rows
first, then down columns. If False, down columns first, then along
rows.
:return: figure_object: Instance of `matplotlib.figure.Figure`.
:return: axes_objects_2d_list: 2-D list, where each item is an instance of
`matplotlib.axes._subplots.AxesSubplot`.
:raises: ValueError: if `colour_map_object_by_panel` or
`colour_norm_object_by_panel` has different length than number of
panels.
"""
error_checking.assert_is_numpy_array(field_matrix, num_dimensions=3)
error_checking.assert_is_boolean(row_major)
if row_major:
order_string = 'C'
else:
order_string = 'F'
num_panels = field_matrix.shape[2]
if panel_names is None:
panel_names = [None] * num_panels
if plot_colour_bar_by_panel is None:
plot_colour_bar_by_panel = numpy.full(num_panels, True, dtype=bool)
these_expected_dim = numpy.array([num_panels], dtype=int)
error_checking.assert_is_numpy_array(
numpy.array(panel_names), exact_dimensions=these_expected_dim)
error_checking.assert_is_numpy_array(
numpy.array(field_name_by_panel), exact_dimensions=these_expected_dim)
error_checking.assert_is_boolean_numpy_array(plot_colour_bar_by_panel)
error_checking.assert_is_numpy_array(
plot_colour_bar_by_panel, exact_dimensions=these_expected_dim)
if (colour_map_object_by_panel is None
or colour_norm_object_by_panel is None):
colour_map_object_by_panel = [None] * num_panels
colour_norm_object_by_panel = [None] * num_panels
error_checking.assert_is_list(colour_map_object_by_panel)
error_checking.assert_is_list(colour_norm_object_by_panel)
if len(colour_map_object_by_panel) != num_panels:
error_string = (
'Number of colour maps ({0:d}) should equal number of panels '
'({1:d}).'
).format(len(colour_map_object_by_panel), num_panels)
raise ValueError(error_string)
if len(colour_norm_object_by_panel) != num_panels:
error_string = (
'Number of colour-normalizers ({0:d}) should equal number of panels'
' ({1:d}).'
).format(len(colour_norm_object_by_panel), num_panels)
raise ValueError(error_string)
error_checking.assert_is_integer(num_panel_rows)
error_checking.assert_is_geq(num_panel_rows, 1)
error_checking.assert_is_leq(num_panel_rows, num_panels)
num_panel_columns = int(numpy.ceil(
float(num_panels) / num_panel_rows
))
figure_object, axes_objects_2d_list = plotting_utils.init_panels(
num_panel_rows=num_panel_rows, num_panel_columns=num_panel_columns,
figure_width_inches=figure_width_inches,
figure_height_inches=figure_height_inches)
for k in range(num_panels):
this_panel_row, this_panel_column = numpy.unravel_index(
k, (num_panel_rows, num_panel_columns), order=order_string
)
if plot_colour_bar_by_panel[k]:
this_annotation_string = None
else:
this_annotation_string = panel_names[k]
this_colour_map_object, this_colour_norm_object = (
plot_2d_grid_without_coords(
field_matrix=field_matrix[..., k],
field_name=field_name_by_panel[k],
axes_object=axes_objects_2d_list[
this_panel_row][this_panel_column],
annotation_string=this_annotation_string, font_size=font_size,
colour_map_object=colour_map_object_by_panel[k],
colour_norm_object=colour_norm_object_by_panel[k]
)
)
if not plot_colour_bar_by_panel[k]:
continue
this_extend_min_flag = field_name_by_panel[k] in SHEAR_VORT_DIV_NAMES
plotting_utils.add_colour_bar(
axes_object_or_list=axes_objects_2d_list[
this_panel_row][this_panel_column],
values_to_colour=field_matrix[..., k],
colour_map=this_colour_map_object,
colour_norm_object=this_colour_norm_object,
orientation='horizontal', font_size=font_size,
extend_min=this_extend_min_flag, extend_max=True,
fraction_of_axis_length=0.9)
axes_objects_2d_list[this_panel_row][this_panel_column].set_xlabel(
panel_names[k], fontsize=font_size)
for k in range(num_panel_rows * num_panel_columns):
if k < num_panels:
continue
this_panel_row, this_panel_column = numpy.unravel_index(
k, (num_panel_rows, num_panel_columns), order=order_string
)
axes_objects_2d_list[this_panel_row][this_panel_column].axis('off')
return figure_object, axes_objects_2d_list
def write_standard_file(pickle_file_name,
init_function_name_or_matrices,
list_of_optimized_matrices,
model_file_name,
num_iterations,
learning_rate,
component_type_string,
target_class=None,
layer_name=None,
neuron_indices=None,
channel_index=None,
ideal_activation=None,
storm_ids=None,
storm_times_unix_sec=None):
"""Writes optimized learning examples to Pickle file.
E = number of examples (storm objects)
:param pickle_file_name: Path to output file.
:param init_function_name_or_matrices: See doc for `_do_gradient_descent`.
The only difference here is that, if a function was used, the input
argument must be the function *name* rather than the function itself.
:param list_of_optimized_matrices: List of numpy arrays created by
`_do_gradient_descent`.
:param model_file_name: Path to file with trained model (readable by
`cnn.read_model`).
:param num_iterations: See doc for `_do_gradient_descent`.
:param learning_rate: Same.
:param component_type_string: See doc for
`model_interpretation.check_component_metadata`.
:param target_class: Same.
:param layer_name: Same.
:param neuron_indices: Same.
:param channel_index: Same.
:param ideal_activation: See doc for `optimize_input_for_neuron` or
`optimize_input_for_channel`.
:param storm_ids:
[used only if `init_function_name_or_matrices` is list of matrices]
length-E list of storm IDs (strings).
:param storm_times_unix_sec:
[used only if `init_function_name_or_matrices` is list of matrices]
length-E numpy array of storm times.
:raises: ValueError: if `init_function_name_or_matrices` is a list of numpy
arrays and has a different length than `list_of_optimized_matrices`.
"""
model_interpretation.check_component_metadata(
component_type_string=component_type_string,
target_class=target_class,
layer_name=layer_name,
neuron_indices=neuron_indices,
channel_index=channel_index)
_check_input_args(num_iterations=num_iterations,
learning_rate=learning_rate,
ideal_activation=ideal_activation)
error_checking.assert_is_string(model_file_name)
error_checking.assert_is_list(list_of_optimized_matrices)
if isinstance(init_function_name_or_matrices, str):
num_storm_objects = None
else:
num_init_matrices = len(init_function_name_or_matrices)
num_optimized_matrices = len(list_of_optimized_matrices)
if num_init_matrices != num_optimized_matrices:
error_string = (
'Number of input matrices ({0:d}) should equal number of output'
' matrices ({1:d}).').format(num_init_matrices,
num_optimized_matrices)
raise ValueError(error_string)
error_checking.assert_is_string_list(storm_ids)
error_checking.assert_is_numpy_array(numpy.array(storm_ids),
num_dimensions=1)
num_storm_objects = len(storm_ids)
these_expected_dim = numpy.array([num_storm_objects], dtype=int)
error_checking.assert_is_integer_numpy_array(storm_times_unix_sec)
error_checking.assert_is_numpy_array(
storm_times_unix_sec, exact_dimensions=these_expected_dim)
num_matrices = len(list_of_optimized_matrices)
for i in range(num_matrices):
error_checking.assert_is_numpy_array_without_nan(
list_of_optimized_matrices[i])
if num_storm_objects is not None:
these_expected_dim = numpy.array(
(num_storm_objects, ) +
list_of_optimized_matrices[i].shape[1:],
dtype=int)
error_checking.assert_is_numpy_array(
list_of_optimized_matrices[i],
exact_dimensions=these_expected_dim)
if not isinstance(init_function_name_or_matrices, str):
error_checking.assert_is_numpy_array_without_nan(
init_function_name_or_matrices[i])
these_expected_dim = numpy.array(
list_of_optimized_matrices[i].shape, dtype=int)
error_checking.assert_is_numpy_array(
init_function_name_or_matrices[i],
exact_dimensions=these_expected_dim)
optimization_dict = {
INIT_FUNCTION_KEY: init_function_name_or_matrices,
OPTIMIZED_MATRICES_KEY: list_of_optimized_matrices,
MODEL_FILE_NAME_KEY: model_file_name,
NUM_ITERATIONS_KEY: num_iterations,
LEARNING_RATE_KEY: learning_rate,
COMPONENT_TYPE_KEY: component_type_string,
TARGET_CLASS_KEY: target_class,
LAYER_NAME_KEY: layer_name,
IDEAL_ACTIVATION_KEY: ideal_activation,
NEURON_INDICES_KEY: neuron_indices,
CHANNEL_INDEX_KEY: channel_index,
STORM_IDS_KEY: storm_ids,
STORM_TIMES_KEY: storm_times_unix_sec
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(optimization_dict, pickle_file_handle)
pickle_file_handle.close()
def test_assert_is_list_scalar(self):
"""Checks assert_is_list when input is scalar."""
with self.assertRaises(TypeError):
error_checking.assert_is_list(SINGLE_INTEGER)
def write_pmm_file(pickle_file_name, list_of_mean_input_matrices,
list_of_mean_saliency_matrices, threshold_count_matrix,
standard_saliency_file_name, pmm_metadata_dict):
"""Writes mean saliency map to Pickle file.
This is a mean over many examples, created by PMM (probability-matched
means).
:param pickle_file_name: Path to output file.
:param list_of_mean_input_matrices: Same as input `list_of_input_matrices`
to `write_standard_file`, but without the first axis.
:param list_of_mean_saliency_matrices: Same as input
`list_of_saliency_matrices` to `write_standard_file`, but without the
first axis.
:param threshold_count_matrix: See doc for
`prob_matched_means.run_pmm_many_variables`.
:param standard_saliency_file_name: Path to file with standard saliency
output (readable by `read_standard_file`).
:param pmm_metadata_dict: Dictionary created by
`prob_matched_means.check_input_args`.
:raises: ValueError: if `list_of_mean_input_matrices` and
`list_of_mean_saliency_matrices` have different lengths.
"""
# TODO(thunderhoser): This method currently does not deal with sounding
# pressures.
error_checking.assert_is_string(standard_saliency_file_name)
error_checking.assert_is_list(list_of_mean_input_matrices)
error_checking.assert_is_list(list_of_mean_saliency_matrices)
num_input_matrices = len(list_of_mean_input_matrices)
num_saliency_matrices = len(list_of_mean_saliency_matrices)
if num_input_matrices != num_saliency_matrices:
error_string = (
'Number of input matrices ({0:d}) should equal number of saliency '
'matrices ({1:d}).').format(num_input_matrices,
num_saliency_matrices)
raise ValueError(error_string)
for i in range(num_input_matrices):
error_checking.assert_is_numpy_array_without_nan(
list_of_mean_input_matrices[i])
error_checking.assert_is_numpy_array_without_nan(
list_of_mean_saliency_matrices[i])
these_expected_dim = numpy.array(list_of_mean_input_matrices[i].shape,
dtype=int)
error_checking.assert_is_numpy_array(
list_of_mean_saliency_matrices[i],
exact_dimensions=these_expected_dim)
if threshold_count_matrix is not None:
error_checking.assert_is_integer_numpy_array(threshold_count_matrix)
error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0)
spatial_dimensions = numpy.array(
list_of_mean_input_matrices[0].shape[:-1], dtype=int)
error_checking.assert_is_numpy_array(
threshold_count_matrix, exact_dimensions=spatial_dimensions)
mean_saliency_dict = {
MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices,
MEAN_SALIENCY_MATRICES_KEY: list_of_mean_saliency_matrices,
THRESHOLD_COUNTS_KEY: threshold_count_matrix,
STANDARD_FILE_NAME_KEY: standard_saliency_file_name,
PMM_METADATA_KEY: pmm_metadata_dict
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(mean_saliency_dict, pickle_file_handle)
pickle_file_handle.close()
def test_assert_is_list_true(self):
"""Checks assert_is_list when input is list."""
error_checking.assert_is_list(REAL_NUMBER_LIST)
def plot_saliency_for_sounding(saliency_matrix,
sounding_field_names,
pressure_levels_mb,
colour_map_object,
max_absolute_colour_value,
min_font_size=DEFAULT_MIN_SOUNDING_FONT_SIZE,
max_font_size=DEFAULT_MAX_SOUNDING_FONT_SIZE):
"""Plots saliency for one sounding.
P = number of pressure levels
F = number of fields
:param saliency_matrix: P-by-F numpy array of saliency values.
:param sounding_field_names: length-F list of field names.
:param pressure_levels_mb: length-P list of pressure levels (millibars).
:param colour_map_object: See doc for `plot_2d_grid`.
:param max_absolute_colour_value: Same.
:param min_font_size: Same.
:param max_font_size: Same.
"""
error_checking.assert_is_geq(max_absolute_colour_value, 0.)
max_absolute_colour_value = max([max_absolute_colour_value, 0.001])
error_checking.assert_is_greater_numpy_array(pressure_levels_mb, 0.)
error_checking.assert_is_numpy_array(pressure_levels_mb, num_dimensions=1)
error_checking.assert_is_list(sounding_field_names)
error_checking.assert_is_numpy_array(numpy.array(sounding_field_names),
num_dimensions=1)
num_pressure_levels = len(pressure_levels_mb)
num_sounding_fields = len(sounding_field_names)
error_checking.assert_is_numpy_array_without_nan(saliency_matrix)
error_checking.assert_is_numpy_array(saliency_matrix,
exact_dimensions=numpy.array([
num_pressure_levels,
num_sounding_fields
]))
try:
u_wind_index = sounding_field_names.index(soundings.U_WIND_NAME)
v_wind_index = sounding_field_names.index(soundings.V_WIND_NAME)
plot_wind_barbs = True
except ValueError:
plot_wind_barbs = False
if plot_wind_barbs:
u_wind_saliency_values = saliency_matrix[:, u_wind_index]
v_wind_saliency_values = saliency_matrix[:, v_wind_index]
wind_saliency_magnitudes = numpy.sqrt(u_wind_saliency_values**2 +
v_wind_saliency_values**2)
colour_norm_object = pyplot.Normalize(vmin=0.,
vmax=max_absolute_colour_value)
rgb_matrix_for_wind = colour_map_object(
colour_norm_object(wind_saliency_magnitudes))[..., :-1]
non_wind_flags = numpy.array(
[f not in WIND_COMPONENT_NAMES for f in sounding_field_names],
dtype=bool)
non_wind_indices = numpy.where(non_wind_flags)[0]
saliency_matrix = saliency_matrix[:, non_wind_indices]
sounding_field_names = [
sounding_field_names[k] for k in non_wind_indices
]
sounding_field_names.append(WIND_NAME)
num_sounding_fields = len(sounding_field_names)
rgb_matrix, font_size_matrix = _saliency_to_colour_and_size(
saliency_matrix=saliency_matrix,
colour_map_object=colour_map_object,
max_absolute_colour_value=max_absolute_colour_value,
min_font_size=min_font_size,
max_font_size=max_font_size)
_, axes_object = pyplot.subplots(1,
1,
figsize=(FIGURE_WIDTH_INCHES,
FIGURE_HEIGHT_INCHES))
axes_object.set_facecolor(
plotting_utils.colour_from_numpy_to_tuple(
SOUNDING_SALIENCY_BACKGROUND_COLOUR))
for k in range(num_sounding_fields):
if sounding_field_names[k] == WIND_NAME:
for j in range(num_pressure_levels):
this_vector = numpy.array(
[u_wind_saliency_values[j], v_wind_saliency_values[j]])
this_vector = (WIND_SALIENCY_MULTIPLIER * this_vector /
numpy.linalg.norm(this_vector, ord=2))
this_colour_tuple = plotting_utils.colour_from_numpy_to_tuple(
rgb_matrix_for_wind[j, ...])
axes_object.barbs(k,
pressure_levels_mb[j],
this_vector[0],
this_vector[1],
length=WIND_BARB_LENGTH,
fill_empty=True,
rounding=False,
sizes={'emptybarb': EMPTY_WIND_BARB_RADIUS},
color=this_colour_tuple)
continue
for j in range(num_pressure_levels):
this_colour_tuple = plotting_utils.colour_from_numpy_to_tuple(
rgb_matrix[j, k, ...])
if saliency_matrix[j, k] >= 0:
axes_object.text(k,
pressure_levels_mb[j],
'+',
fontsize=font_size_matrix[j, k],
color=this_colour_tuple,
horizontalalignment='center',
verticalalignment='center')
else:
axes_object.text(k,
pressure_levels_mb[j],
'_',
fontsize=font_size_matrix[j, k],
color=this_colour_tuple,
horizontalalignment='center',
verticalalignment='bottom')
axes_object.set_xlim(-0.5, num_sounding_fields - 0.5)
axes_object.set_ylim(100, 1000)
axes_object.invert_yaxis()
pyplot.yscale('log')
pyplot.minorticks_off()
y_tick_locations = numpy.linspace(100, 1000, num=10, dtype=int)
y_tick_labels = ['{0:d}'.format(p) for p in y_tick_locations]
pyplot.yticks(y_tick_locations, y_tick_labels)
x_tick_locations = numpy.linspace(0,
num_sounding_fields - 1,
num=num_sounding_fields,
dtype=float)
x_tick_labels = [FIELD_NAME_TO_LATEX_DICT[f] for f in sounding_field_names]
pyplot.xticks(x_tick_locations, x_tick_labels)
colour_bar_object = plotting_utils.plot_linear_colour_bar(
axes_object_or_matrix=axes_object,
data_matrix=saliency_matrix,
colour_map_object=colour_map_object,
min_value=0.,
max_value=max_absolute_colour_value,
orientation_string='vertical',
extend_min=True,
extend_max=True)
colour_bar_object.set_label('Saliency (absolute value)')
def permute_one_predictor(
predictor_matrices, separate_heights, matrix_index, predictor_index,
permuted_values=None):
"""Permutes values of one predictor.
Specifically, will permute values of the [j]th predictor in the [i]th
matrix, where i = `matrix_index` and j = `predictor_index`.
T = number of input tensors to the model
E = number of examples
:param predictor_matrices: length-T list of numpy arrays, where the first
axis of each has length E.
:param separate_heights: Boolean flag. If True, for arrays with 3 spatial
dimensions, each predictor/height pair will be shuffled independently.
If False, for arrays with 3 spatial dimensions, each predictor will be
shuffled independently.
:param matrix_index: See discussion above.
:param predictor_index: See discussion above.
:param permuted_values: numpy array of permuted values with which to replace
clean values. If None, permuted values will be created randomly on the
fly.
:return: predictor_matrices: Same as input but after permutation.
:return: permuted_values: numpy array of permuted values with which clean
values were replaced.
"""
# Check input args.
error_checking.assert_is_list(predictor_matrices)
for this_matrix in predictor_matrices:
error_checking.assert_is_numpy_array_without_nan(this_matrix)
error_checking.assert_is_boolean(separate_heights)
error_checking.assert_is_integer(matrix_index)
error_checking.assert_is_geq(matrix_index, 0)
error_checking.assert_is_integer(predictor_index)
error_checking.assert_is_geq(predictor_index, 0)
if permuted_values is not None:
error_checking.assert_is_numpy_array_without_nan(permuted_values)
# Do dirty work.
i = matrix_index
j = predictor_index
num_spatial_dim = len(predictor_matrices[i].shape) - 2
if num_spatial_dim == 3 and separate_heights:
predictor_matrices[i], original_shape = flatten_last_two_dim(
predictor_matrices[i]
)
else:
original_shape = None
if permuted_values is None:
random_indices = numpy.random.permutation(
predictor_matrices[i].shape[0]
)
predictor_matrices[i][..., j] = (
predictor_matrices[i][random_indices, ..., j]
)
# predictor_matrices[i][..., j] = numpy.take(
# predictor_matrices[i][..., j],
# indices=numpy.random.permutation(predictor_matrices[i].shape[0]),
# axis=0
# )
else:
predictor_matrices[i][..., j] = permuted_values
permuted_values = predictor_matrices[i][..., j]
if original_shape is not None:
predictor_matrices[i] = numpy.reshape(
predictor_matrices[i], original_shape, order='F'
)
return predictor_matrices, permuted_values
