def run_guided_gradcam(orig_model_object, list_of_input_matrices, target_layer_name, class_activation_matrix, new_model_object=None): """Runs guided Grad-CAM. M = number of rows in grid N = number of columns in grid C = number of channels :param orig_model_object: Original model (trained instance of `keras.models.Model` or `keras.models.Sequential`). :param list_of_input_matrices: See doc for `run_gradcam`. :param target_layer_name: Same. :param class_activation_matrix: Same. :param new_model_object: New model (created by `_change_backprop_function`), to be used for guided backprop. :return: ggradcam_output_matrix: M-by-N-by-C numpy array of output values. :return: new_model_object: See input doc. """ # Check input args. error_checking.assert_is_string(target_layer_name) error_checking.assert_is_list(list_of_input_matrices) error_checking.assert_is_numpy_array_without_nan(class_activation_matrix) for q in range(len(list_of_input_matrices)): error_checking.assert_is_numpy_array(list_of_input_matrices[q]) if list_of_input_matrices[q].shape[0] != 1: list_of_input_matrices[q] = numpy.expand_dims( list_of_input_matrices[q], axis=0) # Do the dirty work. if new_model_object is None: _register_guided_backprop() new_model_object = _change_backprop_function( model_object=orig_model_object) input_index = _find_relevant_input_matrix( list_of_input_matrices=list_of_input_matrices, num_spatial_dim=len(class_activation_matrix.shape)) saliency_function = _make_saliency_function(model_object=new_model_object, layer_name=target_layer_name, input_index=input_index) saliency_matrix = saliency_function(list_of_input_matrices + [0])[0] print 'Minimum saliency = {0:.4e} ... max saliency = {1:.4e}'.format( numpy.min(saliency_matrix), numpy.max(saliency_matrix)) ggradcam_output_matrix = saliency_matrix * class_activation_matrix[ ..., numpy.newaxis] ggradcam_output_matrix = ggradcam_output_matrix[0, ...] # ggradcam_output_matrix = _normalize_guided_gradcam_output( # ggradcam_output_matrix[0, ...]) return ggradcam_output_matrix, new_model_object
def _check_region_dict(list_of_cam_matrices, region_dict, pmm_flag): """Error-checks dictionary with regions of interest. :param list_of_cam_matrices: See doc for `_check_in_and_out_matrices`. :param region_dict: Dictionary with the following keys. region_dict['list_of_mask_matrices']: Same as `list_of_cam_matrices`, except that all numpy arrays. Grid cells marked True are in a region of interest. region_dict['list_of_polygon_objects']: Triple-nested list of polygons (instances of `shapely.geometry.Polygon`), demarcating regions of interest. list_of_polygon_objects[i][j][k] is the [k]th region of interest for the [j]th input matrix for the [i]th example. region_dict['percentile_threshold']: Percentile threshold used to create regions of interest. This is applied separately to each class-activation matrix for each example. region_dict['min_class_activation']: Minimum class activation used to create regions of interest. For a grid cell to be in a region of interest, it must meet both this and the percentile threshold. :param pmm_flag: Boolean flag. If True, inputs should contain PMM (probability-matched mean) composites. """ error_checking.assert_is_geq(region_dict[PERCENTILE_THRESHOLD_KEY], 50.) error_checking.assert_is_less_than( region_dict[PERCENTILE_THRESHOLD_KEY], 100. ) error_checking.assert_is_greater(region_dict[MIN_CLASS_ACTIVATION_KEY], 0.) list_of_mask_matrices = region_dict[MASK_MATRICES_KEY] list_of_polygon_objects = region_dict[POLYGON_OBJECTS_KEY] num_input_matrices = len(list_of_cam_matrices) assert len(list_of_mask_matrices) == num_input_matrices assert len(list_of_polygon_objects) == num_input_matrices for j in range(num_input_matrices): if list_of_cam_matrices[j] is None: assert list_of_mask_matrices[j] is None continue error_checking.assert_is_boolean_numpy_array(list_of_mask_matrices[j]) these_expected_dim = numpy.array( list_of_cam_matrices[j].shape, dtype=int ) error_checking.assert_is_numpy_array( list_of_mask_matrices[j], exact_dimensions=these_expected_dim ) if pmm_flag: num_examples = 1 else: num_examples = list_of_cam_matrices[j].shape[0] assert len(list_of_polygon_objects[j]) == num_examples for i in range(num_examples): error_checking.assert_is_list(list_of_polygon_objects[j][i])
def _check_in_and_out_matrices(predictor_matrices, num_examples=None, saliency_matrices=None): """Error-checks input and output matrices. T = number of input tensors to the model E = number of examples (storm objects) :param predictor_matrices: length-T list of predictor matrices. Each item must be a numpy array. :param num_examples: E in the above discussion. The first axis of each array must have length E. If you don't know the number of examples, leave this as None. :param saliency_matrices: Same as `predictor_matrices` but with saliency values. :raises: ValueError: if `predictor_matrices` and `saliency_matrices` have different lengths. """ error_checking.assert_is_list(predictor_matrices) num_matrices = len(predictor_matrices) if saliency_matrices is None: saliency_matrices = [None] * num_matrices error_checking.assert_is_list(saliency_matrices) num_saliency_matrices = len(saliency_matrices) if num_matrices != num_saliency_matrices: error_string = ( 'Number of predictor matrices ({0:d}) should = number of saliency ' 'matrices ({1:d}).').format(num_matrices, num_saliency_matrices) raise ValueError(error_string) for i in range(num_matrices): error_checking.assert_is_numpy_array_without_nan(predictor_matrices[i]) if num_examples is not None: these_expected_dim = numpy.array( (num_examples, ) + predictor_matrices[i].shape[1:], dtype=int) error_checking.assert_is_numpy_array( predictor_matrices[i], exact_dimensions=these_expected_dim) if saliency_matrices[i] is not None: error_checking.assert_is_numpy_array_without_nan( saliency_matrices[i]) these_expected_dim = numpy.array(predictor_matrices[i].shape, dtype=int) error_checking.assert_is_numpy_array( saliency_matrices[i], exact_dimensions=these_expected_dim)
def apply_model_for_each_class(orig_class_probability_matrix, observed_labels, model_object_by_class): """Applies isotonic-regression model for each class. :param orig_class_probability_matrix: See documentation for `train_model_for_each_class`. :param observed_labels: Same. :param model_object_by_class: Same. :return: new_class_probability_matrix: Calibrated version of `orig_class_probability_matrix`. :raises: ValueError: if number of models != number of columns in `orig_class_probability_matrix`. """ _check_evaluation_pairs( class_probability_matrix=orig_class_probability_matrix, observed_labels=observed_labels) error_checking.assert_is_list(model_object_by_class) num_classes = orig_class_probability_matrix.shape[1] if len(model_object_by_class) != num_classes: error_string = ( 'Number of models ({0:d}) should = number of columns in ' 'orig_class_probability_matrix ({1:d}).').format( len(model_object_by_class), num_classes) raise ValueError(error_string) num_examples = orig_class_probability_matrix.shape[0] new_class_probability_matrix = numpy.full((num_examples, num_classes), numpy.nan) for k in range(num_classes): print 'Applying isotonic-regression model for class {0:d}...'.format(k) new_class_probability_matrix[:, k] = model_object_by_class[k].predict( orig_class_probability_matrix[:, k]) # Ensure that sum of class probabilities = 1 for each example. for i in range(num_examples): new_class_probability_matrix[i, :] = ( new_class_probability_matrix[i, :] / numpy.sum(new_class_probability_matrix[i, :])) _check_evaluation_pairs( class_probability_matrix=new_class_probability_matrix, observed_labels=observed_labels) return new_class_probability_matrix
def _polygon_list_to_vertex_list(polygon_objects_grid_coords): """Converts list of polygons to list of vertices. V = total number of vertices :param polygon_objects_grid_coords: List of polygons created by `polygons_from_pixel_to_grid_coords`. :return: vertex_rows: length-V numpy array of row coordinates. :return: vertex_columns: length-V numpy array of column coordinates. :return: vertex_to_polygon_indices: length-V numpy array of indices. If vertex_to_polygon_indices[i] = j, the [i]th vertex comes from the [j]th input polygon. """ error_checking.assert_is_list(polygon_objects_grid_coords) num_polygons = len(polygon_objects_grid_coords) if num_polygons > 0: error_checking.assert_is_numpy_array(numpy.array( polygon_objects_grid_coords, dtype=object), num_dimensions=1) vertex_rows = [] vertex_columns = [] vertex_to_polygon_indices = [] for i in range(num_polygons): this_num_vertices = len(polygon_objects_grid_coords[i].exterior.xy[1]) vertex_rows += polygon_objects_grid_coords[i].exterior.xy[1] vertex_columns += polygon_objects_grid_coords[i].exterior.xy[0] vertex_to_polygon_indices += [i] * this_num_vertices if i == num_polygons - 1: continue vertex_rows += [numpy.nan] vertex_columns += [numpy.nan] vertex_to_polygon_indices += [-1] return (numpy.array(vertex_rows), numpy.array(vertex_columns), numpy.array(vertex_to_polygon_indices, dtype=int))
def write_pmm_file(pickle_file_name, list_of_mean_input_matrices, list_of_mean_optimized_matrices, mean_initial_activation, mean_final_activation, threshold_count_matrix, model_file_name, standard_bwo_file_name, pmm_metadata_dict, monte_carlo_dict=None): """Writes mean backwards-optimized map to Pickle file. This is a mean over many examples, created by PMM (probability-matched means). T = number of input tensors to the model :param pickle_file_name: Path to output file. :param list_of_mean_input_matrices: length-T list of numpy arrays, where list_of_mean_input_matrices[i] is the mean (over many examples) of the [i]th input tensor to the model. list_of_mean_input_matrices[i] should have the same dimensions as the [i]th input tensor, except without the first axis. :param list_of_mean_optimized_matrices: Same as `list_of_mean_input_matrices` (and with the same dimensions), but for optimized learning examples. In other words, `list_of_mean_input_matrices` contains the mean input and `list_of_mean_optimized_matrices` contains the mean output. :param mean_initial_activation: Mean initial activation of relevant model component (before backwards optimization). :param mean_fimal_activation: Mean final activation of relevant model component (after backwards optimization). :param threshold_count_matrix: See doc for `prob_matched_means.run_pmm_many_variables`. :param model_file_name: Path to file with trained model (readable by `cnn.read_model`). :param standard_bwo_file_name: Path to file with standard backwards-optimization output (readable by `read_standard_file`). :param pmm_metadata_dict: Dictionary created by `prob_matched_means.check_input_args`. :param monte_carlo_dict: Dictionary with results of Monte Carlo significance test. Must contain keys listed in `monte_carlo.check_output`, plus the following. monte_carlo_dict['baseline_file_name']: Path to saliency file for baseline set (readable by `read_standard_file`), against which the trial set (contained in `standard_bwo_file_name`) was compared. :raises: ValueError: if `list_of_mean_input_matrices` and `list_of_mean_optimized_matrices` have different lengths. """ error_checking.assert_is_string(model_file_name) error_checking.assert_is_string(standard_bwo_file_name) error_checking.assert_is_list(list_of_mean_input_matrices) error_checking.assert_is_list(list_of_mean_optimized_matrices) num_input_matrices = len(list_of_mean_input_matrices) num_output_matrices = len(list_of_mean_optimized_matrices) if num_input_matrices != num_output_matrices: error_string = ( 'Number of input matrices ({0:d}) should equal number of output ' 'matrices ({1:d}).').format(num_input_matrices, num_output_matrices) raise ValueError(error_string) for i in range(num_input_matrices): error_checking.assert_is_numpy_array_without_nan( list_of_mean_input_matrices[i]) error_checking.assert_is_numpy_array_without_nan( list_of_mean_optimized_matrices[i]) these_expected_dim = numpy.array(list_of_mean_input_matrices[i].shape, dtype=int) error_checking.assert_is_numpy_array( list_of_mean_optimized_matrices[i], exact_dimensions=these_expected_dim) if threshold_count_matrix is not None: error_checking.assert_is_integer_numpy_array(threshold_count_matrix) error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0) spatial_dimensions = numpy.array( list_of_mean_input_matrices[0].shape[:-1], dtype=int) error_checking.assert_is_numpy_array( threshold_count_matrix, exact_dimensions=spatial_dimensions) if monte_carlo_dict is not None: monte_carlo.check_output(monte_carlo_dict) error_checking.assert_is_string( monte_carlo_dict[monte_carlo.BASELINE_FILE_KEY]) for i in range(num_input_matrices): assert numpy.allclose( list_of_mean_optimized_matrices[i], monte_carlo_dict[monte_carlo.TRIAL_PMM_MATRICES_KEY][i], atol=TOLERANCE) mean_optimization_dict = { MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices, MEAN_OPTIMIZED_MATRICES_KEY: list_of_mean_optimized_matrices, MEAN_INITIAL_ACTIVATION_KEY: mean_initial_activation, MEAN_FINAL_ACTIVATION_KEY: mean_final_activation, THRESHOLD_COUNTS_KEY: threshold_count_matrix, MODEL_FILE_KEY: model_file_name, STANDARD_FILE_NAME_KEY: standard_bwo_file_name, PMM_METADATA_KEY: pmm_metadata_dict, MONTE_CARLO_DICT_KEY: monte_carlo_dict } file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name) pickle_file_handle = open(pickle_file_name, 'wb') pickle.dump(mean_optimization_dict, pickle_file_handle) pickle_file_handle.close()
def _check_in_and_out_matrices(predictor_matrices, num_examples=None, cam_matrices=None, guided_cam_matrices=None): """Error-checks input and output matrices. T = number of input tensors to the model E = number of examples (storm objects) :param predictor_matrices: length-T list of predictor matrices. Each item must be a numpy array. :param num_examples: E in the above discussion. The first axis of each array must have length E. If you don't know the number of examples, leave this as None. :param cam_matrices: length-T list of class-activation maps. cam_matrices[i] is the class-activation map for predictor_matrices[i] and should have the same dimensions as predictor_matrices[i], except without the last dimension, which is the channel dimension. If the [i]th tensor is not connected to the target layer, cam_matrices[i] should be None. :param guided_cam_matrices: length-T list of guided class-activation maps. Same as cam_matrices[i], except that cam_matrices[i] should have the same dimensions as predictor_matrices[i], including the last dimension. :raises: ValueError: if `cam_matrices` or `guided_cam_matrices` has different length than `predictor_matrices`. """ error_checking.assert_is_list(predictor_matrices) num_matrices = len(predictor_matrices) if cam_matrices is None: cam_matrices = [None] * num_matrices if guided_cam_matrices is None: guided_cam_matrices = [None] * num_matrices num_cam_matrices = len(cam_matrices) num_guided_cam_matrices = len(guided_cam_matrices) if num_matrices != num_cam_matrices: error_string = ( 'Number of predictor matrices ({0:d}) should = number of CAM ' 'matrices ({1:d}).').format(num_matrices, num_cam_matrices) raise ValueError(error_string) if num_matrices != num_guided_cam_matrices: error_string = ( 'Number of predictor matrices ({0:d}) should = number of guided-CAM' ' matrices ({1:d}).').format(num_matrices, num_guided_cam_matrices) raise ValueError(error_string) for i in range(num_matrices): error_checking.assert_is_numpy_array_without_nan(predictor_matrices[i]) if num_examples is not None: these_expected_dim = numpy.array( (num_examples, ) + predictor_matrices[i].shape[1:], dtype=int) error_checking.assert_is_numpy_array( predictor_matrices[i], exact_dimensions=these_expected_dim) if cam_matrices[i] is not None: error_checking.assert_is_numpy_array_without_nan(cam_matrices[i]) these_expected_dim = numpy.array(predictor_matrices[i].shape[:-1], dtype=int) error_checking.assert_is_numpy_array( cam_matrices[i], exact_dimensions=these_expected_dim) if guided_cam_matrices[i] is not None: error_checking.assert_is_numpy_array_without_nan( guided_cam_matrices[i]) these_expected_dim = numpy.array(predictor_matrices[i].shape, dtype=int) error_checking.assert_is_numpy_array( guided_cam_matrices[i], exact_dimensions=these_expected_dim)
def test_assert_is_list_numpy_array(self): """Checks assert_is_list when input is numpy array.""" with self.assertRaises(TypeError): error_checking.assert_is_list(REAL_NUMPY_ARRAY)
def test_assert_is_list_tuple(self): """Checks assert_is_list when input is tuple.""" with self.assertRaises(TypeError): error_checking.assert_is_list(REAL_NUMBER_TUPLE)
def write_predictions( pickle_file_name, denorm_recon_radar_matrices, full_storm_id_strings, storm_times_unix_sec, mse_by_example, upconvnet_file_name): """Writes predictions (reconstructed radar images) to Pickle file. E = number of examples :param pickle_file_name: Path to output file. :param denorm_recon_radar_matrices: 1-D list of denormalized, reconstructed radar images. Each item must be a 4-D or 5-D numpy array where the first axis has length E. :param full_storm_id_strings: length-E list of storm IDs. :param storm_times_unix_sec: length-E numpy array of valid times. :param mse_by_example: length-E numpy array of mean squared errors (in normalized, not physical, units). :param upconvnet_file_name: Path to upconvnet that generated the reconstructed images (readable by `cnn.read_model`). """ error_checking.assert_is_string_list(full_storm_id_strings) error_checking.assert_is_numpy_array( numpy.array(full_storm_id_strings), num_dimensions=1 ) num_examples = len(full_storm_id_strings) error_checking.assert_is_list(denorm_recon_radar_matrices) for this_matrix in denorm_recon_radar_matrices: error_checking.assert_is_numpy_array_without_nan(this_matrix) this_num_dimensions = len(this_matrix.shape) error_checking.assert_is_geq(this_num_dimensions, 4) error_checking.assert_is_leq(this_num_dimensions, 5) these_expected_dim = numpy.array( (num_examples,) + this_matrix.shape[1:], dtype=int ) error_checking.assert_is_numpy_array( this_matrix, exact_dimensions=these_expected_dim) these_expected_dim = numpy.array([num_examples], dtype=int) error_checking.assert_is_integer_numpy_array(storm_times_unix_sec) error_checking.assert_is_numpy_array( storm_times_unix_sec, exact_dimensions=these_expected_dim) error_checking.assert_is_geq_numpy_array(mse_by_example, 0.) error_checking.assert_is_numpy_array( mse_by_example, exact_dimensions=these_expected_dim) error_checking.assert_is_string(upconvnet_file_name) prediction_dict = { RECON_IMAGE_MATRICES_KEY: denorm_recon_radar_matrices, MEAN_SQUARED_ERRORS_KEY: mse_by_example, STORM_TIMES_KEY: storm_times_unix_sec, FULL_STORM_IDS_KEY: full_storm_id_strings, UPCONVNET_FILE_KEY: upconvnet_file_name } file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name) pickle_file_handle = open(pickle_file_name, 'wb') pickle.dump(prediction_dict, pickle_file_handle) pickle_file_handle.close()
def write_standard_file(pickle_file_name, list_of_input_matrices, list_of_saliency_matrices, storm_ids, storm_times_unix_sec, model_file_name, saliency_metadata_dict, sounding_pressure_matrix_pascals=None): """Writes saliency maps (one per example) to Pickle file. T = number of input tensors to the model E = number of examples (storm objects) H = number of height levels per sounding :param pickle_file_name: Path to output file. :param list_of_input_matrices: length-T list of numpy arrays, containing predictors (inputs to the model). The first dimension of each array must have length E. :param list_of_saliency_matrices: length-T list of numpy arrays, containing saliency values. list_of_saliency_matrices[i] must have the same dimensions as list_of_input_matrices[i]. :param storm_ids: length-E list of storm IDs (strings). :param storm_times_unix_sec: length-E numpy array of storm times. :param model_file_name: Path to file with trained model (readable by `cnn.read_model`). :param saliency_metadata_dict: Dictionary created by `check_metadata`. :param sounding_pressure_matrix_pascals: E-by-H numpy array of pressure levels in soundings. Useful only when the model input contains soundings with no pressure, because it is needed to plot soundings. :raises: ValueError: if `list_of_input_matrices` and `list_of_saliency_matrices` have different lengths. """ error_checking.assert_is_string(model_file_name) error_checking.assert_is_string_list(storm_ids) error_checking.assert_is_numpy_array(numpy.array(storm_ids), num_dimensions=1) num_storm_objects = len(storm_ids) these_expected_dim = numpy.array([num_storm_objects], dtype=int) error_checking.assert_is_integer_numpy_array(storm_times_unix_sec) error_checking.assert_is_numpy_array(storm_times_unix_sec, exact_dimensions=these_expected_dim) error_checking.assert_is_list(list_of_input_matrices) error_checking.assert_is_list(list_of_saliency_matrices) num_input_matrices = len(list_of_input_matrices) num_saliency_matrices = len(list_of_saliency_matrices) if num_input_matrices != num_saliency_matrices: error_string = ( 'Number of input matrices ({0:d}) should equal number of saliency ' 'matrices ({1:d}).').format(num_input_matrices, num_saliency_matrices) raise ValueError(error_string) for i in range(num_input_matrices): error_checking.assert_is_numpy_array_without_nan( list_of_input_matrices[i]) error_checking.assert_is_numpy_array_without_nan( list_of_saliency_matrices[i]) these_expected_dim = numpy.array( (num_storm_objects, ) + list_of_input_matrices[i].shape[1:], dtype=int) error_checking.assert_is_numpy_array( list_of_input_matrices[i], exact_dimensions=these_expected_dim) these_expected_dim = numpy.array(list_of_input_matrices[i].shape, dtype=int) error_checking.assert_is_numpy_array( list_of_saliency_matrices[i], exact_dimensions=these_expected_dim) if sounding_pressure_matrix_pascals is not None: error_checking.assert_is_numpy_array_without_nan( sounding_pressure_matrix_pascals) error_checking.assert_is_greater_numpy_array( sounding_pressure_matrix_pascals, 0.) error_checking.assert_is_numpy_array(sounding_pressure_matrix_pascals, num_dimensions=2) these_expected_dim = numpy.array( (num_storm_objects, ) + sounding_pressure_matrix_pascals.shape[1:], dtype=int) error_checking.assert_is_numpy_array( sounding_pressure_matrix_pascals, exact_dimensions=these_expected_dim) saliency_dict = { INPUT_MATRICES_KEY: list_of_input_matrices, SALIENCY_MATRICES_KEY: list_of_saliency_matrices, STORM_IDS_KEY: storm_ids, STORM_TIMES_KEY: storm_times_unix_sec, MODEL_FILE_NAME_KEY: model_file_name, COMPONENT_TYPE_KEY: saliency_metadata_dict[COMPONENT_TYPE_KEY], TARGET_CLASS_KEY: saliency_metadata_dict[TARGET_CLASS_KEY], LAYER_NAME_KEY: saliency_metadata_dict[LAYER_NAME_KEY], IDEAL_ACTIVATION_KEY: saliency_metadata_dict[IDEAL_ACTIVATION_KEY], NEURON_INDICES_KEY: saliency_metadata_dict[NEURON_INDICES_KEY], CHANNEL_INDEX_KEY: saliency_metadata_dict[CHANNEL_INDEX_KEY], SOUNDING_PRESSURES_KEY: sounding_pressure_matrix_pascals } file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name) pickle_file_handle = open(pickle_file_name, 'wb') pickle.dump(saliency_dict, pickle_file_handle) pickle_file_handle.close()
def write_pmm_file(pickle_file_name, list_of_mean_input_matrices, list_of_mean_optimized_matrices, threshold_count_matrix, standard_bwo_file_name, pmm_metadata_dict): """Writes mean backwards-optimized map to Pickle file. This is a mean over many examples, created by PMM (probability-matched means). T = number of input tensors to the model :param pickle_file_name: Path to output file. :param list_of_mean_input_matrices: length-T list of numpy arrays, where list_of_mean_input_matrices[i] is the mean (over many examples) of the [i]th input tensor to the model. list_of_mean_input_matrices[i] should have the same dimensions as the [i]th input tensor, except without the first axis. :param list_of_mean_optimized_matrices: Same as `list_of_mean_input_matrices` (and with the same dimensions), but for optimized learning examples. In other words, `list_of_mean_input_matrices` contains the mean input and `list_of_mean_optimized_matrices` contains the mean output. :param threshold_count_matrix: See doc for `prob_matched_means.run_pmm_many_variables`. :param standard_bwo_file_name: Path to file with standard backwards-optimization output (readable by `read_standard_file`). :param pmm_metadata_dict: Dictionary created by `prob_matched_means.check_input_args`. :raises: ValueError: if `list_of_mean_input_matrices` and `list_of_mean_optimized_matrices` have different lengths. """ error_checking.assert_is_string(standard_bwo_file_name) error_checking.assert_is_list(list_of_mean_input_matrices) error_checking.assert_is_list(list_of_mean_optimized_matrices) num_input_matrices = len(list_of_mean_input_matrices) num_output_matrices = len(list_of_mean_optimized_matrices) if num_input_matrices != num_output_matrices: error_string = ( 'Number of input matrices ({0:d}) should equal number of output ' 'matrices ({1:d}).').format(num_input_matrices, num_output_matrices) raise ValueError(error_string) for i in range(num_input_matrices): error_checking.assert_is_numpy_array_without_nan( list_of_mean_input_matrices[i]) error_checking.assert_is_numpy_array_without_nan( list_of_mean_optimized_matrices[i]) these_expected_dim = numpy.array(list_of_mean_input_matrices[i].shape, dtype=int) error_checking.assert_is_numpy_array( list_of_mean_optimized_matrices[i], exact_dimensions=these_expected_dim) if threshold_count_matrix is not None: error_checking.assert_is_integer_numpy_array(threshold_count_matrix) error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0) spatial_dimensions = numpy.array( list_of_mean_input_matrices[0].shape[:-1], dtype=int) error_checking.assert_is_numpy_array( threshold_count_matrix, exact_dimensions=spatial_dimensions) mean_optimization_dict = { MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices, MEAN_OPTIMIZED_MATRICES_KEY: list_of_mean_optimized_matrices, THRESHOLD_COUNTS_KEY: threshold_count_matrix, STANDARD_FILE_NAME_KEY: standard_bwo_file_name, PMM_METADATA_KEY: pmm_metadata_dict } file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name) pickle_file_handle = open(pickle_file_name, 'wb') pickle.dump(mean_optimization_dict, pickle_file_handle) pickle_file_handle.close()
def plot_many_soundings(list_of_metpy_dictionaries, title_strings, num_panel_rows, output_file_name, temp_directory_name=None, option_dict=None): """Creates paneled figure with many soundings. N = number of soundings to plot :param list_of_metpy_dictionaries: length-N list of dictionaries. Each dictionary must satisfy the input format for `sounding_dict_for_metpy` in `plot_sounding`. :param title_strings: length-N list of titles. :param num_panel_rows: Number of rows in paneled figure. :param output_file_name: Path to output (image) file. :param temp_directory_name: Name of temporary directory. Each panel will be stored here, then deleted after the panels have been concatenated into the final image. If `temp_directory_name is None`, will use the default temp directory on the local machine. :param option_dict: See doc for `plot_sounding`. """ error_checking.assert_is_numpy_array(numpy.array(title_strings), num_dimensions=1) num_soundings = len(title_strings) error_checking.assert_is_list(list_of_metpy_dictionaries) error_checking.assert_is_geq(len(list_of_metpy_dictionaries), num_soundings) error_checking.assert_is_leq(len(list_of_metpy_dictionaries), num_soundings) error_checking.assert_is_integer(num_panel_rows) error_checking.assert_is_geq(num_panel_rows, 1) error_checking.assert_is_leq(num_panel_rows, num_soundings) file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) if temp_directory_name is not None: file_system_utils.mkdir_recursive_if_necessary( directory_name=temp_directory_name) temp_file_names = [None] * num_soundings num_panel_columns = int(numpy.ceil(float(num_soundings) / num_panel_rows)) for i in range(num_panel_rows): for j in range(num_panel_columns): this_sounding_index = i * num_panel_columns + j if this_sounding_index >= num_soundings: break plot_sounding(sounding_dict_for_metpy=list_of_metpy_dictionaries[ this_sounding_index], title_string=title_strings[this_sounding_index], option_dict=option_dict) temp_file_names[this_sounding_index] = '{0:s}.jpg'.format( tempfile.NamedTemporaryFile(dir=temp_directory_name, delete=False).name) print('Saving sounding to: "{0:s}"...'.format( temp_file_names[this_sounding_index])) pyplot.savefig(temp_file_names[this_sounding_index], dpi=DOTS_PER_INCH) pyplot.close() imagemagick_utils.trim_whitespace( input_file_name=temp_file_names[this_sounding_index], output_file_name=temp_file_names[this_sounding_index], border_width_pixels=SINGLE_IMAGE_BORDER_WIDTH_PX) imagemagick_utils.resize_image( input_file_name=temp_file_names[this_sounding_index], output_file_name=temp_file_names[this_sounding_index], output_size_pixels=SINGLE_IMAGE_SIZE_PX) print('Concatenating panels into one figure: "{0:s}"...'.format( output_file_name)) imagemagick_utils.concatenate_images( input_file_names=temp_file_names, output_file_name=output_file_name, num_panel_rows=num_panel_rows, num_panel_columns=num_panel_columns, border_width_pixels=PANELED_IMAGE_BORDER_WIDTH_PX) for i in range(num_soundings): os.remove(temp_file_names[i])
def plot_many_2d_grids_without_coords(field_matrix, field_name_by_panel, num_panel_rows=None, figure_object=None, axes_object_matrix=None, panel_names=None, colour_map_object_by_panel=None, colour_norm_object_by_panel=None, plot_colour_bar_by_panel=None, font_size=DEFAULT_FONT_SIZE, row_major=True): """Plots 2-D colour map in each panel (one per field/height pair). M = number of rows in spatial grid N = number of columns in spatial grid P = number of panels (field/height pairs) This method uses the default colour scheme for each radar field. If `num_panel_rows is None`, this method needs arguments `figure_object` and `axes_object_matrix` -- and vice-versa. :param field_matrix: M-by-N-by-P numpy array of radar values. :param field_name_by_panel: length-P list of field names. :param num_panel_rows: Number of rows in paneled figure (different than M, which is number of rows in spatial grid). :param figure_object: See doc for `plotting_utils.create_paneled_figure`. :param axes_object_matrix: See above. :param panel_names: length-P list of panel names (will be printed at bottoms of panels). If you do not want panel names, make this None. :param colour_map_object_by_panel: length-P list of `matplotlib.pyplot.cm` objects. If this is None, the default will be used for each field. :param colour_norm_object_by_panel: length-P list of `matplotlib.colors.BoundaryNorm` objects. If this is None, the default will be used for each field. :param plot_colour_bar_by_panel: length-P numpy array of Boolean flags. If plot_colour_bar_by_panel[k] = True, horizontal colour bar will be plotted under [k]th panel. If you want to plot colour bar for every panel, leave this as None. :param font_size: Font size. :param row_major: Boolean flag. If True, panels will be filled along rows first, then down columns. If False, down columns first, then along rows. :return: figure_object: See doc for `plotting_utils.create_paneled_figure`. :return: axes_object_matrix: Same. :raises: ValueError: if `colour_map_object_by_panel` or `colour_norm_object_by_panel` has different length than number of panels. """ error_checking.assert_is_boolean(row_major) error_checking.assert_is_numpy_array(field_matrix, num_dimensions=3) num_panels = field_matrix.shape[2] if panel_names is None: panel_names = [None] * num_panels if plot_colour_bar_by_panel is None: plot_colour_bar_by_panel = numpy.full(num_panels, True, dtype=bool) these_expected_dim = numpy.array([num_panels], dtype=int) error_checking.assert_is_numpy_array(numpy.array(panel_names), exact_dimensions=these_expected_dim) error_checking.assert_is_numpy_array(numpy.array(field_name_by_panel), exact_dimensions=these_expected_dim) error_checking.assert_is_boolean_numpy_array(plot_colour_bar_by_panel) error_checking.assert_is_numpy_array(plot_colour_bar_by_panel, exact_dimensions=these_expected_dim) if (colour_map_object_by_panel is None or colour_norm_object_by_panel is None): colour_map_object_by_panel = [None] * num_panels colour_norm_object_by_panel = [None] * num_panels error_checking.assert_is_list(colour_map_object_by_panel) error_checking.assert_is_list(colour_norm_object_by_panel) if len(colour_map_object_by_panel) != num_panels: error_string = ( 'Number of colour maps ({0:d}) should equal number of panels ' '({1:d}).').format(len(colour_map_object_by_panel), num_panels) raise ValueError(error_string) if len(colour_norm_object_by_panel) != num_panels: error_string = ( 'Number of colour-normalizers ({0:d}) should equal number of panels' ' ({1:d}).').format(len(colour_norm_object_by_panel), num_panels) raise ValueError(error_string) if figure_object is None: error_checking.assert_is_integer(num_panel_rows) error_checking.assert_is_geq(num_panel_rows, 1) error_checking.assert_is_leq(num_panel_rows, num_panels) num_panel_columns = int(numpy.ceil(float(num_panels) / num_panel_rows)) figure_object, axes_object_matrix = ( plotting_utils.create_paneled_figure(num_rows=num_panel_rows, num_columns=num_panel_columns, shared_x_axis=False, shared_y_axis=False, keep_aspect_ratio=True)) else: error_checking.assert_is_numpy_array(axes_object_matrix, num_dimensions=2) num_panel_rows = axes_object_matrix.shape[0] num_panel_columns = axes_object_matrix.shape[1] if row_major: order_string = 'C' else: order_string = 'F' for k in range(num_panels): this_panel_row, this_panel_column = numpy.unravel_index( k, (num_panel_rows, num_panel_columns), order=order_string) # this_colour_map_object, this_colour_norm_object = ( # plot_2d_grid_without_coords( # field_matrix=field_matrix[..., k], # field_name=field_name_by_panel[k], # axes_object=axes_object_matrix[ # this_panel_row, this_panel_column], # annotation_string=panel_names[k], font_size=font_size, # colour_map_object=colour_map_object_by_panel[k], # colour_norm_object=colour_norm_object_by_panel[k] # ) # ) this_colour_map_object, this_colour_norm_object = ( plot_2d_grid_without_coords( field_matrix=field_matrix[..., k], field_name=field_name_by_panel[k], axes_object=axes_object_matrix[this_panel_row, this_panel_column], annotation_string=None, font_size=font_size, colour_map_object=colour_map_object_by_panel[k], colour_norm_object=colour_norm_object_by_panel[k])) if not plot_colour_bar_by_panel[k]: continue this_extend_min_flag = field_name_by_panel[k] in SHEAR_VORT_DIV_NAMES this_colour_bar_object = plotting_utils.plot_colour_bar( axes_object_or_matrix=axes_object_matrix[this_panel_row, this_panel_column], data_matrix=field_matrix[..., k], colour_map_object=this_colour_map_object, colour_norm_object=this_colour_norm_object, orientation_string='horizontal', extend_min=this_extend_min_flag, extend_max=True, fraction_of_axis_length=0.75, font_size=font_size) this_colour_bar_object.set_label(panel_names[k].replace('\n', '; '), fontsize=font_size, fontweight='bold') for k in range(num_panel_rows * num_panel_columns): if k < num_panels: continue this_panel_row, this_panel_column = numpy.unravel_index( k, (num_panel_rows, num_panel_columns), order=order_string) axes_object_matrix[this_panel_row, this_panel_column].axis('off') return figure_object, axes_object_matrix
def _check_in_and_out_matrices( list_of_input_matrices, num_examples, list_of_cam_matrices=None, list_of_guided_cam_matrices=None): """Error-checks input and output matrices for Grad-CAM. T = number of input tensors to the model :param list_of_input_matrices: length-T list of numpy arrays, containing only one example (storm object). list_of_input_matrices[i] must have the same dimensions as the [i]th input tensor to the model. :param num_examples: Number of examples expected. If num_examples <= 0, this method will allow the example dimension to be missing from the matrices. :param list_of_cam_matrices: length-T list of numpy arrays. list_of_cam_matrices[i] will be the class-activation map for list_of_input_matrices[i]. It will have the same dimensions as list_of_input_matrices[i], just without the channel dimension. If the target layer is not connected to the [i]th input tensor, list_of_cam_matrices[i] will be None. :param list_of_guided_cam_matrices: length-T list of numpy arrays. list_of_guided_cam_matrices[i] will be the guided class-activation map for list_of_input_matrices[i] and will have the same dimensions as list_of_input_matrices[i]. If the target layer is not connected to the [i]th input tensor, list_of_guided_cam_matrices[i] will be None. :return: list_of_input_matrices: Same as input but guaranteed to have example dimensions. """ error_checking.assert_is_list(list_of_input_matrices) num_input_matrices = len(list_of_input_matrices) if list_of_cam_matrices is None: list_of_cam_matrices = [None] * num_input_matrices if list_of_guided_cam_matrices is None: list_of_guided_cam_matrices = [None] * num_input_matrices assert len(list_of_cam_matrices) == num_input_matrices assert len(list_of_guided_cam_matrices) == num_input_matrices for i in range(num_input_matrices): error_checking.assert_is_numpy_array_without_nan( list_of_input_matrices[i] ) if num_examples == 1 and list_of_input_matrices[i].shape[0] != 1: list_of_input_matrices[i] = numpy.expand_dims( list_of_input_matrices[i], axis=0 ) if num_examples > 0: assert list_of_input_matrices[i].shape[0] == num_examples if list_of_cam_matrices[i] is not None: error_checking.assert_is_numpy_array_without_nan( list_of_cam_matrices[i] ) these_expected_dim = numpy.array( list_of_input_matrices[i].shape[:-1], dtype=int ) error_checking.assert_is_numpy_array( list_of_cam_matrices[i], exact_dimensions=these_expected_dim ) if list_of_guided_cam_matrices[i] is not None: error_checking.assert_is_numpy_array_without_nan( list_of_guided_cam_matrices[i] ) these_expected_dim = numpy.array( list_of_input_matrices[i].shape, dtype=int ) error_checking.assert_is_numpy_array( list_of_guided_cam_matrices[i], exact_dimensions=these_expected_dim )
def polygons_from_pixel_to_grid_coords(polygon_objects_pixel_coords, num_grid_rows, num_grid_columns, num_pixel_rows, num_pixel_columns, num_panel_rows, num_panel_columns): """Converts polygons from pixel coordinates to grid coordinates. The input args `num_grid_rows` and `num_grid_columns` are the number of rows and columns in the data grid. These are *not* the same as the number of pixel rows and columns in the image. This method assumes that the image contains one or more panels with gridded data. Each panel may contain a different variable, but they must all contain the same grid, with the same aspect ratio and no whitespace border (between the panels or around the outside of the image). N = number of polygons :param polygon_objects_pixel_coords: length-N list of polygons (instances of `shapely.geometry.Polygon`) with vertices in pixel coordinates, where the top-left corner is x = y = 0. :param num_grid_rows: Number of rows in grid. :param num_grid_columns: Number of columns in grid. :param num_pixel_rows: Number of pixel rows in image. :param num_pixel_columns: Number of pixel columns in image. :param num_panel_rows: Number of panel rows in image. :param num_panel_columns: Number of panel columns in image. :return: polygon_objects_grid_coords: length-N list of polygons (instances of `shapely.geometry.Polygon`) with vertices in grid coordinates, where the bottom-left corner is x = y = 0. :return: panel_row_by_polygon: length-N numpy array of panel rows. If panel_rows[k] = i, the [k]th polygon corresponds to a grid in the [i]th panel row, where the top row is the 0th. :return: panel_column_by_polygon: length-N numpy array of panel columns. If panel_columns[k] = j, the [k]th polygon corresponds to a grid in the [j]th panel column, where the left column is the 0th. :raises: ValueError: if one polygon is in multiple panels. """ error_checking.assert_is_list(polygon_objects_pixel_coords) num_polygons = len(polygon_objects_pixel_coords) if num_polygons == 0: empty_array = numpy.array([], dtype=int) return polygon_objects_pixel_coords, empty_array, empty_array error_checking.assert_is_numpy_array(numpy.array( polygon_objects_pixel_coords, dtype=object), num_dimensions=1) polygon_objects_grid_coords = [None] * num_polygons panel_row_by_polygon = [None] * num_polygons panel_column_by_polygon = [None] * num_polygons for k in range(num_polygons): these_grid_columns, these_panel_columns = pixel_columns_to_grid_columns( pixel_column_by_vertex=numpy.array( polygon_objects_pixel_coords[k].exterior.xy[0]), num_pixel_columns=num_pixel_columns, num_panel_columns=num_panel_columns, num_grid_columns=num_grid_columns, assert_same_panel=True) panel_column_by_polygon[k] = these_panel_columns[0] these_grid_rows, these_panel_rows = pixel_rows_to_grid_rows( pixel_row_by_vertex=numpy.array( polygon_objects_pixel_coords[k].exterior.xy[1]), num_pixel_rows=num_pixel_rows, num_panel_rows=num_panel_rows, num_grid_rows=num_grid_rows, assert_same_panel=True) panel_row_by_polygon[k] = these_panel_rows[0] polygon_objects_grid_coords[k] = ( polygons.vertex_arrays_to_polygon_object( exterior_x_coords=these_grid_columns, exterior_y_coords=these_grid_rows)) panel_row_by_polygon = numpy.array(panel_row_by_polygon, dtype=int) panel_column_by_polygon = numpy.array(panel_column_by_polygon, dtype=int) return (polygon_objects_grid_coords, panel_row_by_polygon, panel_column_by_polygon)
def write_standard_file(pickle_file_name, list_of_input_matrices, class_activation_matrix, ggradcam_output_matrix, model_file_name, storm_ids, storm_times_unix_sec, target_class, target_layer_name, sounding_pressure_matrix_pascals=None): """Writes class-activation maps (one for each example) to Pickle file. T = number of input tensors to the model E = number of examples (storm objects) H = number of height levels per sounding :param pickle_file_name: Path to output file. :param list_of_input_matrices: length-T list of numpy arrays, containing denormalized input data for all examples. The first axis of each array must have length E. :param class_activation_matrix: numpy array, where class_activation_matrix[i, ...] is the activation matrix for the [i]th example, created by `run_gradcam`. :param ggradcam_output_matrix: numpy array, where ggradcam_output_matrix[i, ...] contains output of guided Grad-CAM for the [i]th example, created by `run_guided_gradcam`. :param model_file_name: Path to file with trained CNN (readable by `cnn.read_model`). :param storm_ids: length-E list of storm IDs (strings). :param storm_times_unix_sec: length-E numpy array of storm times. :param target_class: See doc for `run_gradcam`. :param target_layer_name: Same. :param sounding_pressure_matrix_pascals: E-by-H numpy array of pressure levels in soundings. Useful when model input contains soundings with no pressure variable, since pressure is needed to plot soundings. :raises: ValueError: if `list_of_input_matrices` and `class_activation_matrix` have non-matching dimensions. """ error_checking.assert_is_string(model_file_name) error_checking.assert_is_integer(target_class) error_checking.assert_is_geq(target_class, 0) error_checking.assert_is_string(target_layer_name) error_checking.assert_is_string_list(storm_ids) error_checking.assert_is_numpy_array(numpy.array(storm_ids), num_dimensions=1) num_storm_objects = len(storm_ids) error_checking.assert_is_integer_numpy_array(storm_times_unix_sec) error_checking.assert_is_numpy_array(storm_times_unix_sec, exact_dimensions=numpy.array( [num_storm_objects])) error_checking.assert_is_numpy_array_without_nan(class_activation_matrix) error_checking.assert_is_geq(len(class_activation_matrix.shape), 2) these_expected_dim = numpy.array( (num_storm_objects, ) + class_activation_matrix.shape[1:], dtype=int) error_checking.assert_is_numpy_array(class_activation_matrix, exact_dimensions=these_expected_dim) error_checking.assert_is_numpy_array_without_nan(ggradcam_output_matrix) num_channels = ggradcam_output_matrix.shape[-1] these_expected_dim = numpy.array(class_activation_matrix.shape + (num_channels, ), dtype=int) error_checking.assert_is_numpy_array(ggradcam_output_matrix, exact_dimensions=these_expected_dim) error_checking.assert_is_list(list_of_input_matrices) for this_input_matrix in list_of_input_matrices: error_checking.assert_is_numpy_array_without_nan(this_input_matrix) these_expected_dim = numpy.array( (num_storm_objects, ) + this_input_matrix.shape[1:], dtype=int) error_checking.assert_is_numpy_array( this_input_matrix, exact_dimensions=these_expected_dim) if sounding_pressure_matrix_pascals is not None: error_checking.assert_is_numpy_array(sounding_pressure_matrix_pascals, num_dimensions=2) these_expected_dim = numpy.array( (num_storm_objects, ) + sounding_pressure_matrix_pascals.shape[1:], dtype=int) error_checking.assert_is_numpy_array( sounding_pressure_matrix_pascals, exact_dimensions=these_expected_dim) gradcam_dict = { INPUT_MATRICES_KEY: list_of_input_matrices, CLASS_ACTIVATIONS_KEY: class_activation_matrix, GUIDED_GRADCAM_KEY: ggradcam_output_matrix, MODEL_FILE_NAME_KEY: model_file_name, STORM_IDS_KEY: storm_ids, STORM_TIMES_KEY: storm_times_unix_sec, TARGET_CLASS_KEY: target_class, TARGET_LAYER_KEY: target_layer_name, SOUNDING_PRESSURES_KEY: sounding_pressure_matrix_pascals } file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name) pickle_file_handle = open(pickle_file_name, 'wb') pickle.dump(gradcam_dict, pickle_file_handle) pickle_file_handle.close()
def write_pmm_file(pickle_file_name, list_of_mean_input_matrices, mean_class_activation_matrix, mean_ggradcam_output_matrix, threshold_count_matrix, standard_gradcam_file_name, pmm_metadata_dict): """Writes mean class-activation map to Pickle file. This is a mean over many examples, created by PMM (probability-matched means). :param pickle_file_name: Path to output file. :param list_of_mean_input_matrices: Same as input `list_of_input_matrices` to `write_standard_file`, but without the first axis. :param mean_class_activation_matrix: Same as input `class_activation_matrix` to `write_standard_file`, but without the first axis. :param mean_ggradcam_output_matrix: Same as input `ggradcam_output_matrix` to `write_standard_file`, but without the first axis. :param threshold_count_matrix: See doc for `prob_matched_means.run_pmm_many_variables`. :param standard_gradcam_file_name: Path to file with standard Grad-CAM output (readable by `read_standard_file`). :param pmm_metadata_dict: Dictionary created by `prob_matched_means.check_input_args`. """ # TODO(thunderhoser): This method currently does not deal with sounding # pressures. error_checking.assert_is_string(standard_gradcam_file_name) error_checking.assert_is_numpy_array_without_nan( mean_class_activation_matrix) spatial_dimensions = numpy.array(mean_class_activation_matrix.shape, dtype=int) error_checking.assert_is_numpy_array_without_nan( mean_ggradcam_output_matrix) num_channels = mean_ggradcam_output_matrix.shape[-1] these_expected_dim = numpy.array(mean_class_activation_matrix.shape + (num_channels, ), dtype=int) error_checking.assert_is_numpy_array(mean_ggradcam_output_matrix, exact_dimensions=these_expected_dim) if threshold_count_matrix is not None: error_checking.assert_is_integer_numpy_array(threshold_count_matrix) error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0) error_checking.assert_is_numpy_array( threshold_count_matrix, exact_dimensions=spatial_dimensions) error_checking.assert_is_list(list_of_mean_input_matrices) for this_input_matrix in list_of_mean_input_matrices: error_checking.assert_is_numpy_array_without_nan(this_input_matrix) mean_gradcam_dict = { MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices, MEAN_CLASS_ACTIVATIONS_KEY: mean_class_activation_matrix, MEAN_GUIDED_GRADCAM_KEY: mean_ggradcam_output_matrix, THRESHOLD_COUNTS_KEY: threshold_count_matrix, STANDARD_FILE_NAME_KEY: standard_gradcam_file_name, PMM_METADATA_KEY: pmm_metadata_dict } file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name) pickle_file_handle = open(pickle_file_name, 'wb') pickle.dump(mean_gradcam_dict, pickle_file_handle) pickle_file_handle.close()
def run_gradcam(model_object, list_of_input_matrices, target_class, target_layer_name): """Runs Grad-CAM. T = number of input tensors to the model :param model_object: Trained instance of `keras.models.Model` or `keras.models.Sequential`. :param list_of_input_matrices: length-T list of numpy arrays, containing only one example (storm object). list_of_input_matrices[i] must have the same dimensions as the [i]th input tensor to the model. :param target_class: Activation maps will be created for this class. Must be an integer in 0...(K - 1), where K = number of classes. :param target_layer_name: Name of target layer. Neuron-importance weights will be based on activations in this layer. :return: class_activation_matrix: Class-activation matrix. Dimensions of this numpy array will be the spatial dimensions of whichever input tensor feeds into the target layer. For example, if the given input tensor is 2-dimensional with M rows and N columns, this array will be M x N. """ # Check input args. error_checking.assert_is_string(target_layer_name) error_checking.assert_is_list(list_of_input_matrices) for q in range(len(list_of_input_matrices)): error_checking.assert_is_numpy_array(list_of_input_matrices[q]) if list_of_input_matrices[q].shape[0] != 1: list_of_input_matrices[q] = numpy.expand_dims( list_of_input_matrices[q], axis=0) # Create loss tensor. output_layer_object = model_object.layers[-1].output num_output_neurons = output_layer_object.get_shape().as_list()[-1] if num_output_neurons == 1: error_checking.assert_is_leq(target_class, 1) print model_object.layers[-1].input if target_class == 1: # loss_tensor = model_object.layers[-1].output[..., 0] loss_tensor = model_object.layers[-1].input[..., 0] else: # loss_tensor = -1 * model_object.layers[-1].output[..., 0] loss_tensor = -1 * model_object.layers[-1].input[..., 0] else: error_checking.assert_is_less_than(target_class, num_output_neurons) # loss_tensor = model_object.layers[-1].output[..., target_class] loss_tensor = model_object.layers[-1].input[..., target_class] # Create gradient function. target_layer_activation_tensor = model_object.get_layer( name=target_layer_name).output gradient_tensor = _compute_gradients(loss_tensor, [target_layer_activation_tensor])[0] gradient_tensor = _normalize_tensor(gradient_tensor) if isinstance(model_object.input, list): list_of_input_tensors = model_object.input else: list_of_input_tensors = [model_object.input] gradient_function = K.function( list_of_input_tensors, [target_layer_activation_tensor, gradient_tensor]) # Evaluate gradient function. target_layer_activation_matrix, gradient_matrix = gradient_function( list_of_input_matrices) target_layer_activation_matrix = target_layer_activation_matrix[0, ...] gradient_matrix = gradient_matrix[0, ...] # Compute class-activation matrix. mean_weight_by_filter = numpy.mean(gradient_matrix, axis=(0, 1)) class_activation_matrix = numpy.ones( target_layer_activation_matrix.shape[:-1]) num_filters = len(mean_weight_by_filter) for m in range(num_filters): class_activation_matrix += (mean_weight_by_filter[m] * target_layer_activation_matrix[..., m]) input_index = _find_relevant_input_matrix( list_of_input_matrices=list_of_input_matrices, num_spatial_dim=len(class_activation_matrix.shape)) spatial_dimensions = numpy.array( list_of_input_matrices[input_index].shape[1:-1], dtype=int) class_activation_matrix = _upsample_cam( class_activation_matrix=class_activation_matrix, new_dimensions=spatial_dimensions) class_activation_matrix[class_activation_matrix < 0.] = 0. # denominator = numpy.maximum(numpy.max(class_activation_matrix), K.epsilon()) # return class_activation_matrix / denominator return class_activation_matrix
def write_pmm_file( pickle_file_name, list_of_mean_input_matrices, list_of_mean_saliency_matrices, threshold_count_matrix, model_file_name, standard_saliency_file_name, pmm_metadata_dict, monte_carlo_dict=None): """Writes mean saliency map to Pickle file. This is a mean over many examples, created by PMM (probability-matched means). :param pickle_file_name: Path to output file. :param list_of_mean_input_matrices: Same as input `list_of_input_matrices` to `write_standard_file`, but without the first axis. :param list_of_mean_saliency_matrices: Same as input `list_of_saliency_matrices` to `write_standard_file`, but without the first axis. :param threshold_count_matrix: See doc for `prob_matched_means.run_pmm_many_variables`. :param model_file_name: Path to file with trained model (readable by `cnn.read_model`). :param standard_saliency_file_name: Path to file with standard saliency output (readable by `read_standard_file`). :param pmm_metadata_dict: Dictionary created by `prob_matched_means.check_input_args`. :param monte_carlo_dict: Dictionary with results of Monte Carlo significance test. Must contain keys listed in `monte_carlo.check_output`, plus the following. monte_carlo_dict['baseline_file_name']: Path to saliency file for baseline set (readable by `read_standard_file`), against which the trial set (contained in `standard_saliency_file_name`) was compared. :raises: ValueError: if `list_of_mean_input_matrices` and `list_of_mean_saliency_matrices` have different lengths. """ # TODO(thunderhoser): This method currently does not deal with sounding # pressures. error_checking.assert_is_string(model_file_name) error_checking.assert_is_string(standard_saliency_file_name) error_checking.assert_is_list(list_of_mean_input_matrices) error_checking.assert_is_list(list_of_mean_saliency_matrices) num_input_matrices = len(list_of_mean_input_matrices) num_saliency_matrices = len(list_of_mean_saliency_matrices) if num_input_matrices != num_saliency_matrices: error_string = ( 'Number of input matrices ({0:d}) should equal number of saliency ' 'matrices ({1:d}).' ).format(num_input_matrices, num_saliency_matrices) raise ValueError(error_string) for i in range(num_input_matrices): error_checking.assert_is_numpy_array_without_nan( list_of_mean_input_matrices[i]) error_checking.assert_is_numpy_array_without_nan( list_of_mean_saliency_matrices[i]) these_expected_dim = numpy.array( list_of_mean_input_matrices[i].shape, dtype=int) error_checking.assert_is_numpy_array( list_of_mean_saliency_matrices[i], exact_dimensions=these_expected_dim) if threshold_count_matrix is not None: error_checking.assert_is_integer_numpy_array(threshold_count_matrix) error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0) spatial_dimensions = numpy.array( list_of_mean_input_matrices[0].shape[:-1], dtype=int) error_checking.assert_is_numpy_array( threshold_count_matrix, exact_dimensions=spatial_dimensions) if monte_carlo_dict is not None: monte_carlo.check_output(monte_carlo_dict) error_checking.assert_is_string( monte_carlo_dict[monte_carlo.BASELINE_FILE_KEY] ) for i in range(num_input_matrices): assert numpy.allclose( list_of_mean_saliency_matrices[i], monte_carlo_dict[monte_carlo.TRIAL_PMM_MATRICES_KEY][i], atol=TOLERANCE ) mean_saliency_dict = { MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices, MEAN_SALIENCY_MATRICES_KEY: list_of_mean_saliency_matrices, THRESHOLD_COUNTS_KEY: threshold_count_matrix, MODEL_FILE_KEY: model_file_name, STANDARD_FILE_NAME_KEY: standard_saliency_file_name, PMM_METADATA_KEY: pmm_metadata_dict, MONTE_CARLO_DICT_KEY: monte_carlo_dict } file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name) pickle_file_handle = open(pickle_file_name, 'wb') pickle.dump(mean_saliency_dict, pickle_file_handle) pickle_file_handle.close()
def plot_many_2d_grids_without_coords( field_matrix, field_name_by_panel, num_panel_rows, panel_names=None, colour_map_object_by_panel=None, colour_norm_object_by_panel=None, plot_colour_bar_by_panel=None, figure_width_inches=DEFAULT_FIGURE_WIDTH_INCHES, figure_height_inches=DEFAULT_FIGURE_HEIGHT_INCHES, font_size=DEFAULT_FONT_SIZE, row_major=True): """Plots 2-D colour map in each panel (one per field/height pair). M = number of rows in spatial grid N = number of columns in spatial grid P = number of panels (field/height pairs) This method uses the default colour scheme for each radar field. :param field_matrix: M-by-N-by-P numpy array of radar values. :param field_name_by_panel: length-P list of field names. :param num_panel_rows: Number of rows in paneled figure (different than M, which is number of rows in spatial grid). :param panel_names: length-P list of panel names (will be printed at bottoms of panels). If you do not want panel names, make this None. :param colour_map_object_by_panel: length-P list of `matplotlib.pyplot.cm` objects. If this is None, the default will be used for each field. :param colour_norm_object_by_panel: length-P list of `matplotlib.colors.BoundaryNorm` objects. If this is None, the default will be used for each field. :param plot_colour_bar_by_panel: length-P numpy array of Boolean flags. If plot_colour_bar_by_panel[k] = True, horizontal colour bar will be plotted under [k]th panel. If you want to plot colour bar for every panel, leave this as None. :param figure_width_inches: Figure width. :param figure_height_inches: Figure height. :param font_size: Font size. :param row_major: Boolean flag. If True, panels will be filled along rows first, then down columns. If False, down columns first, then along rows. :return: figure_object: Instance of `matplotlib.figure.Figure`. :return: axes_objects_2d_list: 2-D list, where each item is an instance of `matplotlib.axes._subplots.AxesSubplot`. :raises: ValueError: if `colour_map_object_by_panel` or `colour_norm_object_by_panel` has different length than number of panels. """ error_checking.assert_is_numpy_array(field_matrix, num_dimensions=3) error_checking.assert_is_boolean(row_major) if row_major: order_string = 'C' else: order_string = 'F' num_panels = field_matrix.shape[2] if panel_names is None: panel_names = [None] * num_panels if plot_colour_bar_by_panel is None: plot_colour_bar_by_panel = numpy.full(num_panels, True, dtype=bool) these_expected_dim = numpy.array([num_panels], dtype=int) error_checking.assert_is_numpy_array( numpy.array(panel_names), exact_dimensions=these_expected_dim) error_checking.assert_is_numpy_array( numpy.array(field_name_by_panel), exact_dimensions=these_expected_dim) error_checking.assert_is_boolean_numpy_array(plot_colour_bar_by_panel) error_checking.assert_is_numpy_array( plot_colour_bar_by_panel, exact_dimensions=these_expected_dim) if (colour_map_object_by_panel is None or colour_norm_object_by_panel is None): colour_map_object_by_panel = [None] * num_panels colour_norm_object_by_panel = [None] * num_panels error_checking.assert_is_list(colour_map_object_by_panel) error_checking.assert_is_list(colour_norm_object_by_panel) if len(colour_map_object_by_panel) != num_panels: error_string = ( 'Number of colour maps ({0:d}) should equal number of panels ' '({1:d}).' ).format(len(colour_map_object_by_panel), num_panels) raise ValueError(error_string) if len(colour_norm_object_by_panel) != num_panels: error_string = ( 'Number of colour-normalizers ({0:d}) should equal number of panels' ' ({1:d}).' ).format(len(colour_norm_object_by_panel), num_panels) raise ValueError(error_string) error_checking.assert_is_integer(num_panel_rows) error_checking.assert_is_geq(num_panel_rows, 1) error_checking.assert_is_leq(num_panel_rows, num_panels) num_panel_columns = int(numpy.ceil( float(num_panels) / num_panel_rows )) figure_object, axes_objects_2d_list = plotting_utils.init_panels( num_panel_rows=num_panel_rows, num_panel_columns=num_panel_columns, figure_width_inches=figure_width_inches, figure_height_inches=figure_height_inches) for k in range(num_panels): this_panel_row, this_panel_column = numpy.unravel_index( k, (num_panel_rows, num_panel_columns), order=order_string ) if plot_colour_bar_by_panel[k]: this_annotation_string = None else: this_annotation_string = panel_names[k] this_colour_map_object, this_colour_norm_object = ( plot_2d_grid_without_coords( field_matrix=field_matrix[..., k], field_name=field_name_by_panel[k], axes_object=axes_objects_2d_list[ this_panel_row][this_panel_column], annotation_string=this_annotation_string, font_size=font_size, colour_map_object=colour_map_object_by_panel[k], colour_norm_object=colour_norm_object_by_panel[k] ) ) if not plot_colour_bar_by_panel[k]: continue this_extend_min_flag = field_name_by_panel[k] in SHEAR_VORT_DIV_NAMES plotting_utils.add_colour_bar( axes_object_or_list=axes_objects_2d_list[ this_panel_row][this_panel_column], values_to_colour=field_matrix[..., k], colour_map=this_colour_map_object, colour_norm_object=this_colour_norm_object, orientation='horizontal', font_size=font_size, extend_min=this_extend_min_flag, extend_max=True, fraction_of_axis_length=0.9) axes_objects_2d_list[this_panel_row][this_panel_column].set_xlabel( panel_names[k], fontsize=font_size) for k in range(num_panel_rows * num_panel_columns): if k < num_panels: continue this_panel_row, this_panel_column = numpy.unravel_index( k, (num_panel_rows, num_panel_columns), order=order_string ) axes_objects_2d_list[this_panel_row][this_panel_column].axis('off') return figure_object, axes_objects_2d_list
def write_standard_file(pickle_file_name, init_function_name_or_matrices, list_of_optimized_matrices, model_file_name, num_iterations, learning_rate, component_type_string, target_class=None, layer_name=None, neuron_indices=None, channel_index=None, ideal_activation=None, storm_ids=None, storm_times_unix_sec=None): """Writes optimized learning examples to Pickle file. E = number of examples (storm objects) :param pickle_file_name: Path to output file. :param init_function_name_or_matrices: See doc for `_do_gradient_descent`. The only difference here is that, if a function was used, the input argument must be the function *name* rather than the function itself. :param list_of_optimized_matrices: List of numpy arrays created by `_do_gradient_descent`. :param model_file_name: Path to file with trained model (readable by `cnn.read_model`). :param num_iterations: See doc for `_do_gradient_descent`. :param learning_rate: Same. :param component_type_string: See doc for `model_interpretation.check_component_metadata`. :param target_class: Same. :param layer_name: Same. :param neuron_indices: Same. :param channel_index: Same. :param ideal_activation: See doc for `optimize_input_for_neuron` or `optimize_input_for_channel`. :param storm_ids: [used only if `init_function_name_or_matrices` is list of matrices] length-E list of storm IDs (strings). :param storm_times_unix_sec: [used only if `init_function_name_or_matrices` is list of matrices] length-E numpy array of storm times. :raises: ValueError: if `init_function_name_or_matrices` is a list of numpy arrays and has a different length than `list_of_optimized_matrices`. """ model_interpretation.check_component_metadata( component_type_string=component_type_string, target_class=target_class, layer_name=layer_name, neuron_indices=neuron_indices, channel_index=channel_index) _check_input_args(num_iterations=num_iterations, learning_rate=learning_rate, ideal_activation=ideal_activation) error_checking.assert_is_string(model_file_name) error_checking.assert_is_list(list_of_optimized_matrices) if isinstance(init_function_name_or_matrices, str): num_storm_objects = None else: num_init_matrices = len(init_function_name_or_matrices) num_optimized_matrices = len(list_of_optimized_matrices) if num_init_matrices != num_optimized_matrices: error_string = ( 'Number of input matrices ({0:d}) should equal number of output' ' matrices ({1:d}).').format(num_init_matrices, num_optimized_matrices) raise ValueError(error_string) error_checking.assert_is_string_list(storm_ids) error_checking.assert_is_numpy_array(numpy.array(storm_ids), num_dimensions=1) num_storm_objects = len(storm_ids) these_expected_dim = numpy.array([num_storm_objects], dtype=int) error_checking.assert_is_integer_numpy_array(storm_times_unix_sec) error_checking.assert_is_numpy_array( storm_times_unix_sec, exact_dimensions=these_expected_dim) num_matrices = len(list_of_optimized_matrices) for i in range(num_matrices): error_checking.assert_is_numpy_array_without_nan( list_of_optimized_matrices[i]) if num_storm_objects is not None: these_expected_dim = numpy.array( (num_storm_objects, ) + list_of_optimized_matrices[i].shape[1:], dtype=int) error_checking.assert_is_numpy_array( list_of_optimized_matrices[i], exact_dimensions=these_expected_dim) if not isinstance(init_function_name_or_matrices, str): error_checking.assert_is_numpy_array_without_nan( init_function_name_or_matrices[i]) these_expected_dim = numpy.array( list_of_optimized_matrices[i].shape, dtype=int) error_checking.assert_is_numpy_array( init_function_name_or_matrices[i], exact_dimensions=these_expected_dim) optimization_dict = { INIT_FUNCTION_KEY: init_function_name_or_matrices, OPTIMIZED_MATRICES_KEY: list_of_optimized_matrices, MODEL_FILE_NAME_KEY: model_file_name, NUM_ITERATIONS_KEY: num_iterations, LEARNING_RATE_KEY: learning_rate, COMPONENT_TYPE_KEY: component_type_string, TARGET_CLASS_KEY: target_class, LAYER_NAME_KEY: layer_name, IDEAL_ACTIVATION_KEY: ideal_activation, NEURON_INDICES_KEY: neuron_indices, CHANNEL_INDEX_KEY: channel_index, STORM_IDS_KEY: storm_ids, STORM_TIMES_KEY: storm_times_unix_sec } file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name) pickle_file_handle = open(pickle_file_name, 'wb') pickle.dump(optimization_dict, pickle_file_handle) pickle_file_handle.close()
def test_assert_is_list_scalar(self): """Checks assert_is_list when input is scalar.""" with self.assertRaises(TypeError): error_checking.assert_is_list(SINGLE_INTEGER)
def write_pmm_file(pickle_file_name, list_of_mean_input_matrices, list_of_mean_saliency_matrices, threshold_count_matrix, standard_saliency_file_name, pmm_metadata_dict): """Writes mean saliency map to Pickle file. This is a mean over many examples, created by PMM (probability-matched means). :param pickle_file_name: Path to output file. :param list_of_mean_input_matrices: Same as input `list_of_input_matrices` to `write_standard_file`, but without the first axis. :param list_of_mean_saliency_matrices: Same as input `list_of_saliency_matrices` to `write_standard_file`, but without the first axis. :param threshold_count_matrix: See doc for `prob_matched_means.run_pmm_many_variables`. :param standard_saliency_file_name: Path to file with standard saliency output (readable by `read_standard_file`). :param pmm_metadata_dict: Dictionary created by `prob_matched_means.check_input_args`. :raises: ValueError: if `list_of_mean_input_matrices` and `list_of_mean_saliency_matrices` have different lengths. """ # TODO(thunderhoser): This method currently does not deal with sounding # pressures. error_checking.assert_is_string(standard_saliency_file_name) error_checking.assert_is_list(list_of_mean_input_matrices) error_checking.assert_is_list(list_of_mean_saliency_matrices) num_input_matrices = len(list_of_mean_input_matrices) num_saliency_matrices = len(list_of_mean_saliency_matrices) if num_input_matrices != num_saliency_matrices: error_string = ( 'Number of input matrices ({0:d}) should equal number of saliency ' 'matrices ({1:d}).').format(num_input_matrices, num_saliency_matrices) raise ValueError(error_string) for i in range(num_input_matrices): error_checking.assert_is_numpy_array_without_nan( list_of_mean_input_matrices[i]) error_checking.assert_is_numpy_array_without_nan( list_of_mean_saliency_matrices[i]) these_expected_dim = numpy.array(list_of_mean_input_matrices[i].shape, dtype=int) error_checking.assert_is_numpy_array( list_of_mean_saliency_matrices[i], exact_dimensions=these_expected_dim) if threshold_count_matrix is not None: error_checking.assert_is_integer_numpy_array(threshold_count_matrix) error_checking.assert_is_geq_numpy_array(threshold_count_matrix, 0) spatial_dimensions = numpy.array( list_of_mean_input_matrices[0].shape[:-1], dtype=int) error_checking.assert_is_numpy_array( threshold_count_matrix, exact_dimensions=spatial_dimensions) mean_saliency_dict = { MEAN_INPUT_MATRICES_KEY: list_of_mean_input_matrices, MEAN_SALIENCY_MATRICES_KEY: list_of_mean_saliency_matrices, THRESHOLD_COUNTS_KEY: threshold_count_matrix, STANDARD_FILE_NAME_KEY: standard_saliency_file_name, PMM_METADATA_KEY: pmm_metadata_dict } file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name) pickle_file_handle = open(pickle_file_name, 'wb') pickle.dump(mean_saliency_dict, pickle_file_handle) pickle_file_handle.close()
def test_assert_is_list_true(self): """Checks assert_is_list when input is list.""" error_checking.assert_is_list(REAL_NUMBER_LIST)
def plot_saliency_for_sounding(saliency_matrix, sounding_field_names, pressure_levels_mb, colour_map_object, max_absolute_colour_value, min_font_size=DEFAULT_MIN_SOUNDING_FONT_SIZE, max_font_size=DEFAULT_MAX_SOUNDING_FONT_SIZE): """Plots saliency for one sounding. P = number of pressure levels F = number of fields :param saliency_matrix: P-by-F numpy array of saliency values. :param sounding_field_names: length-F list of field names. :param pressure_levels_mb: length-P list of pressure levels (millibars). :param colour_map_object: See doc for `plot_2d_grid`. :param max_absolute_colour_value: Same. :param min_font_size: Same. :param max_font_size: Same. """ error_checking.assert_is_geq(max_absolute_colour_value, 0.) max_absolute_colour_value = max([max_absolute_colour_value, 0.001]) error_checking.assert_is_greater_numpy_array(pressure_levels_mb, 0.) error_checking.assert_is_numpy_array(pressure_levels_mb, num_dimensions=1) error_checking.assert_is_list(sounding_field_names) error_checking.assert_is_numpy_array(numpy.array(sounding_field_names), num_dimensions=1) num_pressure_levels = len(pressure_levels_mb) num_sounding_fields = len(sounding_field_names) error_checking.assert_is_numpy_array_without_nan(saliency_matrix) error_checking.assert_is_numpy_array(saliency_matrix, exact_dimensions=numpy.array([ num_pressure_levels, num_sounding_fields ])) try: u_wind_index = sounding_field_names.index(soundings.U_WIND_NAME) v_wind_index = sounding_field_names.index(soundings.V_WIND_NAME) plot_wind_barbs = True except ValueError: plot_wind_barbs = False if plot_wind_barbs: u_wind_saliency_values = saliency_matrix[:, u_wind_index] v_wind_saliency_values = saliency_matrix[:, v_wind_index] wind_saliency_magnitudes = numpy.sqrt(u_wind_saliency_values**2 + v_wind_saliency_values**2) colour_norm_object = pyplot.Normalize(vmin=0., vmax=max_absolute_colour_value) rgb_matrix_for_wind = colour_map_object( colour_norm_object(wind_saliency_magnitudes))[..., :-1] non_wind_flags = numpy.array( [f not in WIND_COMPONENT_NAMES for f in sounding_field_names], dtype=bool) non_wind_indices = numpy.where(non_wind_flags)[0] saliency_matrix = saliency_matrix[:, non_wind_indices] sounding_field_names = [ sounding_field_names[k] for k in non_wind_indices ] sounding_field_names.append(WIND_NAME) num_sounding_fields = len(sounding_field_names) rgb_matrix, font_size_matrix = _saliency_to_colour_and_size( saliency_matrix=saliency_matrix, colour_map_object=colour_map_object, max_absolute_colour_value=max_absolute_colour_value, min_font_size=min_font_size, max_font_size=max_font_size) _, axes_object = pyplot.subplots(1, 1, figsize=(FIGURE_WIDTH_INCHES, FIGURE_HEIGHT_INCHES)) axes_object.set_facecolor( plotting_utils.colour_from_numpy_to_tuple( SOUNDING_SALIENCY_BACKGROUND_COLOUR)) for k in range(num_sounding_fields): if sounding_field_names[k] == WIND_NAME: for j in range(num_pressure_levels): this_vector = numpy.array( [u_wind_saliency_values[j], v_wind_saliency_values[j]]) this_vector = (WIND_SALIENCY_MULTIPLIER * this_vector / numpy.linalg.norm(this_vector, ord=2)) this_colour_tuple = plotting_utils.colour_from_numpy_to_tuple( rgb_matrix_for_wind[j, ...]) axes_object.barbs(k, pressure_levels_mb[j], this_vector[0], this_vector[1], length=WIND_BARB_LENGTH, fill_empty=True, rounding=False, sizes={'emptybarb': EMPTY_WIND_BARB_RADIUS}, color=this_colour_tuple) continue for j in range(num_pressure_levels): this_colour_tuple = plotting_utils.colour_from_numpy_to_tuple( rgb_matrix[j, k, ...]) if saliency_matrix[j, k] >= 0: axes_object.text(k, pressure_levels_mb[j], '+', fontsize=font_size_matrix[j, k], color=this_colour_tuple, horizontalalignment='center', verticalalignment='center') else: axes_object.text(k, pressure_levels_mb[j], '_', fontsize=font_size_matrix[j, k], color=this_colour_tuple, horizontalalignment='center', verticalalignment='bottom') axes_object.set_xlim(-0.5, num_sounding_fields - 0.5) axes_object.set_ylim(100, 1000) axes_object.invert_yaxis() pyplot.yscale('log') pyplot.minorticks_off() y_tick_locations = numpy.linspace(100, 1000, num=10, dtype=int) y_tick_labels = ['{0:d}'.format(p) for p in y_tick_locations] pyplot.yticks(y_tick_locations, y_tick_labels) x_tick_locations = numpy.linspace(0, num_sounding_fields - 1, num=num_sounding_fields, dtype=float) x_tick_labels = [FIELD_NAME_TO_LATEX_DICT[f] for f in sounding_field_names] pyplot.xticks(x_tick_locations, x_tick_labels) colour_bar_object = plotting_utils.plot_linear_colour_bar( axes_object_or_matrix=axes_object, data_matrix=saliency_matrix, colour_map_object=colour_map_object, min_value=0., max_value=max_absolute_colour_value, orientation_string='vertical', extend_min=True, extend_max=True) colour_bar_object.set_label('Saliency (absolute value)')
def permute_one_predictor( predictor_matrices, separate_heights, matrix_index, predictor_index, permuted_values=None): """Permutes values of one predictor. Specifically, will permute values of the [j]th predictor in the [i]th matrix, where i = `matrix_index` and j = `predictor_index`. T = number of input tensors to the model E = number of examples :param predictor_matrices: length-T list of numpy arrays, where the first axis of each has length E. :param separate_heights: Boolean flag. If True, for arrays with 3 spatial dimensions, each predictor/height pair will be shuffled independently. If False, for arrays with 3 spatial dimensions, each predictor will be shuffled independently. :param matrix_index: See discussion above. :param predictor_index: See discussion above. :param permuted_values: numpy array of permuted values with which to replace clean values. If None, permuted values will be created randomly on the fly. :return: predictor_matrices: Same as input but after permutation. :return: permuted_values: numpy array of permuted values with which clean values were replaced. """ # Check input args. error_checking.assert_is_list(predictor_matrices) for this_matrix in predictor_matrices: error_checking.assert_is_numpy_array_without_nan(this_matrix) error_checking.assert_is_boolean(separate_heights) error_checking.assert_is_integer(matrix_index) error_checking.assert_is_geq(matrix_index, 0) error_checking.assert_is_integer(predictor_index) error_checking.assert_is_geq(predictor_index, 0) if permuted_values is not None: error_checking.assert_is_numpy_array_without_nan(permuted_values) # Do dirty work. i = matrix_index j = predictor_index num_spatial_dim = len(predictor_matrices[i].shape) - 2 if num_spatial_dim == 3 and separate_heights: predictor_matrices[i], original_shape = flatten_last_two_dim( predictor_matrices[i] ) else: original_shape = None if permuted_values is None: random_indices = numpy.random.permutation( predictor_matrices[i].shape[0] ) predictor_matrices[i][..., j] = ( predictor_matrices[i][random_indices, ..., j] ) # predictor_matrices[i][..., j] = numpy.take( # predictor_matrices[i][..., j], # indices=numpy.random.permutation(predictor_matrices[i].shape[0]), # axis=0 # ) else: predictor_matrices[i][..., j] = permuted_values permuted_values = predictor_matrices[i][..., j] if original_shape is not None: predictor_matrices[i] = numpy.reshape( predictor_matrices[i], original_shape, order='F' ) return predictor_matrices, permuted_values