def _smooth_maps(saliency_matrices, smoothing_radius_grid_cells): """Smooths saliency maps via Gaussian filter. T = number of input tensors to the model :param saliency_matrices: length-T list of numpy arrays. :param smoothing_radius_grid_cells: e-folding radius (number of grid cells). :return: saliency_matrices: Smoothed version of input. """ print(('Smoothing saliency maps with Gaussian filter (e-folding radius of ' '{0:.1f} grid cells)...').format(smoothing_radius_grid_cells)) num_matrices = len(saliency_matrices) num_examples = saliency_matrices[0].shape[0] for j in range(num_matrices): this_num_channels = saliency_matrices[j].shape[-1] for i in range(num_examples): for k in range(this_num_channels): saliency_matrices[j][i, ..., k] = ( general_utils.apply_gaussian_filter( input_matrix=saliency_matrices[j][i, ..., k], e_folding_radius_grid_cells=smoothing_radius_grid_cells )) return saliency_matrices
def _smooth_maps(cam_matrices, smoothing_radius_grid_cells): """Smooths class-activation maps via Gaussian filter. T = number of input tensors to the model :param cam_matrices: length-T list of numpy arrays. :param smoothing_radius_grid_cells: e-folding radius (number of grid cells). :return: cam_matrices: Smoothed version of input. """ print(( 'Smoothing class-activation maps with Gaussian filter (e-folding radius' ' of {0:.1f} grid cells)...').format(smoothing_radius_grid_cells)) num_matrices = len(cam_matrices) for j in range(num_matrices): if cam_matrices[j] is None: continue num_examples = cam_matrices[j].shape[0] for i in range(num_examples): cam_matrices[j][i, ...] = general_utils.apply_gaussian_filter( input_matrix=cam_matrices[j][i, ...], e_folding_radius_grid_cells=smoothing_radius_grid_cells) return cam_matrices
def _smooth_maps(cam_matrices, guided_cam_matrices, smoothing_radius_grid_cells): """Smooths guided and unguided class-activation maps, using Gaussian filter. T = number of input tensors to the model :param cam_matrices: length-T list of numpy arrays with unguided class- activation maps (CAMs). :param guided_cam_matrices: length-T list of numpy arrays with guided CAMs. :param smoothing_radius_grid_cells: e-folding radius (number of grid cells). :return: cam_matrices: Smoothed version of input. :return: guided_cam_matrices: Smoothed version of input. """ print(( 'Smoothing guided and unguided CAMs with Gaussian filter (e-folding ' 'radius of {0:.1f} grid cells)...' ).format( smoothing_radius_grid_cells )) num_matrices = len(cam_matrices) for j in range(num_matrices): if cam_matrices[j] is None: continue num_examples = cam_matrices[j].shape[0] this_num_channels = guided_cam_matrices[j].shape[-1] for i in range(num_examples): cam_matrices[j][i, ...] = general_utils.apply_gaussian_filter( input_matrix=cam_matrices[j][i, ...], e_folding_radius_grid_cells=smoothing_radius_grid_cells ) for k in range(this_num_channels): guided_cam_matrices[j][i, ..., k] = ( general_utils.apply_gaussian_filter( input_matrix=guided_cam_matrices[j][i, ..., k], e_folding_radius_grid_cells=smoothing_radius_grid_cells ) ) return cam_matrices, guided_cam_matrices
def _read_one_composite(saliency_file_name, smoothing_radius_grid_cells): """Reads saliency map for one composite. E = number of examples M = number of rows in grid N = number of columns in grid H = number of heights in grid F = number of radar fields :param saliency_file_name: Path to input file (will be read by `saliency.read_file`). :param smoothing_radius_grid_cells: Radius for Gaussian smoother, used only for saliency map. :return: mean_radar_matrix: E-by-M-by-N-by-H-by-F numpy array with mean radar fields. :return: mean_saliency_matrix: E-by-M-by-N-by-H-by-F numpy array with mean saliency fields. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. """ print('Reading data from: "{0:s}"...'.format(saliency_file_name)) saliency_dict = saliency_maps.read_file(saliency_file_name)[0] mean_radar_matrix = numpy.expand_dims( saliency_dict[saliency_maps.MEAN_PREDICTOR_MATRICES_KEY][0], axis=0) mean_saliency_matrix = numpy.expand_dims( saliency_dict[saliency_maps.MEAN_SALIENCY_MATRICES_KEY][0], axis=0) if smoothing_radius_grid_cells is not None: print(( 'Smoothing saliency maps with Gaussian filter (e-folding radius of ' '{0:.1f} grid cells)...').format(smoothing_radius_grid_cells)) num_fields = mean_radar_matrix.shape[-1] for k in range(num_fields): mean_saliency_matrix[0, ..., k] = ( general_utils.apply_gaussian_filter( input_matrix=mean_saliency_matrix[0, ..., k], e_folding_radius_grid_cells=smoothing_radius_grid_cells)) model_file_name = saliency_dict[saliency_maps.MODEL_FILE_KEY] model_metafile_name = cnn.find_metafile(model_file_name) print('Reading CNN metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] good_indices = numpy.array([ numpy.where( training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] == h)[0][0] for h in RADAR_HEIGHTS_M_AGL ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices, :] mean_saliency_matrix = mean_saliency_matrix[..., good_indices, :] good_indices = numpy.array([ training_option_dict[trainval_io.RADAR_FIELDS_KEY].index(f) for f in RADAR_FIELD_NAMES ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices] mean_saliency_matrix = mean_saliency_matrix[..., good_indices] training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = RADAR_HEIGHTS_M_AGL training_option_dict[trainval_io.RADAR_FIELDS_KEY] = RADAR_FIELD_NAMES training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = None model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict return mean_radar_matrix, mean_saliency_matrix, model_metadata_dict
def _run(evaluation_dir_name, smoothing_radius_grid_cells, score_colour_map_name, num_ex_colour_map_name, max_colour_percentile, output_dir_name): """Plots spatially subset model evaluation. This is effectively the main method. :param evaluation_dir_name: See documentation at top of file. :param smoothing_radius_grid_cells: Same. :param score_colour_map_name: Same. :param num_ex_colour_map_name: Same. :param max_colour_percentile: Same. :param output_dir_name: Same. """ if smoothing_radius_grid_cells <= 0: smoothing_radius_grid_cells = None score_colour_map_object = pyplot.get_cmap(score_colour_map_name) num_ex_colour_map_object = pyplot.get_cmap(num_ex_colour_map_name) error_checking.assert_is_geq(max_colour_percentile, 90.) error_checking.assert_is_leq(max_colour_percentile, 100.) grid_metafile_name = grids.find_equidistant_metafile( directory_name=evaluation_dir_name, raise_error_if_missing=True) print('Reading grid metadata from: "{0:s}"...'.format(grid_metafile_name)) grid_metadata_dict = grids.read_equidistant_metafile(grid_metafile_name) print(SEPARATOR_STRING) num_grid_rows = len(grid_metadata_dict[grids.Y_COORDS_KEY]) num_grid_columns = len(grid_metadata_dict[grids.X_COORDS_KEY]) auc_matrix = numpy.full((num_grid_rows, num_grid_columns), numpy.nan) csi_matrix = numpy.full((num_grid_rows, num_grid_columns), numpy.nan) pod_matrix = numpy.full((num_grid_rows, num_grid_columns), numpy.nan) far_matrix = numpy.full((num_grid_rows, num_grid_columns), numpy.nan) num_examples_matrix = numpy.full((num_grid_rows, num_grid_columns), 0, dtype=int) num_positive_examples_matrix = numpy.full( (num_grid_rows, num_grid_columns), 0, dtype=int) for i in range(num_grid_rows): for j in range(num_grid_columns): this_eval_file_name = model_eval.find_file( directory_name=evaluation_dir_name, grid_row=i, grid_column=j, raise_error_if_missing=False) if not os.path.isfile(this_eval_file_name): warning_string = ( 'Cannot find file (this may or may not be a problem). ' 'Expected at: "{0:s}"').format(this_eval_file_name) warnings.warn(warning_string) continue print('Reading data from: "{0:s}"...'.format(this_eval_file_name)) this_evaluation_dict = model_eval.read_evaluation( this_eval_file_name) num_examples_matrix[i, j] = len( this_evaluation_dict[model_eval.OBSERVED_LABELS_KEY]) num_positive_examples_matrix[i, j] = numpy.sum( this_evaluation_dict[model_eval.OBSERVED_LABELS_KEY]) this_evaluation_table = this_evaluation_dict[ model_eval.EVALUATION_TABLE_KEY] auc_matrix[i, j] = numpy.nanmean( this_evaluation_table[model_eval.AUC_KEY].values) csi_matrix[i, j] = numpy.nanmean( this_evaluation_table[model_eval.CSI_KEY].values) pod_matrix[i, j] = numpy.nanmean( this_evaluation_table[model_eval.POD_KEY].values) far_matrix[i, j] = 1. - numpy.nanmean( this_evaluation_table[model_eval.SUCCESS_RATIO_KEY].values) print(SEPARATOR_STRING) auc_matrix[num_positive_examples_matrix == 0] = numpy.nan csi_matrix[num_positive_examples_matrix == 0] = numpy.nan pod_matrix[num_positive_examples_matrix == 0] = numpy.nan far_matrix[num_positive_examples_matrix == 0] = numpy.nan if smoothing_radius_grid_cells is not None: print(( 'Applying Gaussian smoother with e-folding radius of {0:.1f} grid ' 'cells...').format(smoothing_radius_grid_cells)) orig_num_examples_matrix = num_examples_matrix + 0 num_examples_matrix = general_utils.apply_gaussian_filter( input_matrix=num_examples_matrix.astype(float), e_folding_radius_grid_cells=smoothing_radius_grid_cells) num_examples_matrix = numpy.round(num_examples_matrix).astype(int) num_examples_matrix[orig_num_examples_matrix == 0] = 0 # HACK num_positive_examples_matrix = general_utils.apply_gaussian_filter( input_matrix=num_positive_examples_matrix.astype(float), e_folding_radius_grid_cells=smoothing_radius_grid_cells) num_positive_examples_matrix = ( numpy.round(num_positive_examples_matrix).astype(int)) num_positive_examples_matrix[num_examples_matrix == 0] = 0 auc_matrix = general_utils.apply_gaussian_filter( input_matrix=ge_utils.fill_nans(auc_matrix), e_folding_radius_grid_cells=smoothing_radius_grid_cells) csi_matrix = general_utils.apply_gaussian_filter( input_matrix=ge_utils.fill_nans(csi_matrix), e_folding_radius_grid_cells=smoothing_radius_grid_cells) pod_matrix = general_utils.apply_gaussian_filter( input_matrix=ge_utils.fill_nans(pod_matrix), e_folding_radius_grid_cells=smoothing_radius_grid_cells) far_matrix = general_utils.apply_gaussian_filter( input_matrix=ge_utils.fill_nans(far_matrix), e_folding_radius_grid_cells=smoothing_radius_grid_cells) auc_matrix[num_positive_examples_matrix == 0] = numpy.nan csi_matrix[num_positive_examples_matrix == 0] = numpy.nan pod_matrix[num_positive_examples_matrix == 0] = numpy.nan far_matrix[num_positive_examples_matrix == 0] = numpy.nan panel_file_names = [] file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) # Plot number of examples. this_data_matrix = numpy.maximum(numpy.log10(num_examples_matrix), 0.) this_data_matrix[this_data_matrix == 0] = numpy.nan max_colour_value = numpy.nanpercentile(this_data_matrix, max_colour_percentile) figure_object, axes_object = _plot_one_value( data_matrix=this_data_matrix, grid_metadata_dict=grid_metadata_dict, colour_map_object=num_ex_colour_map_object, min_colour_value=0., max_colour_value=max_colour_value, plot_cbar_min_arrow=False, plot_cbar_max_arrow=True, log_scale=True) axes_object.set_title(r'Number of examples') plotting_utils.label_axes(axes_object=axes_object, label_string='(a)') panel_file_names.append('{0:s}/num_examples.jpg'.format(output_dir_name)) print('Saving figure to: "{0:s}"...'.format(panel_file_names[-1])) figure_object.savefig(panel_file_names[-1], dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object) # Plot number of positive examples. this_data_matrix = num_positive_examples_matrix.astype(float) this_data_matrix[this_data_matrix == 0] = numpy.nan max_colour_value = numpy.nanpercentile(this_data_matrix, max_colour_percentile) min_colour_value = numpy.nanpercentile(this_data_matrix, 100. - max_colour_percentile) figure_object, axes_object = _plot_one_value( data_matrix=this_data_matrix, grid_metadata_dict=grid_metadata_dict, colour_map_object=num_ex_colour_map_object, min_colour_value=min_colour_value, max_colour_value=max_colour_value, plot_cbar_min_arrow=True, plot_cbar_max_arrow=True) axes_object.set_title('Number of tornadic examples') plotting_utils.label_axes(axes_object=axes_object, label_string='(b)') panel_file_names.append( '{0:s}/num_positive_examples.jpg'.format(output_dir_name)) print('Saving figure to: "{0:s}"...'.format(panel_file_names[-1])) figure_object.savefig(panel_file_names[-1], dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object) # Plot AUC. max_colour_value = numpy.nanpercentile(auc_matrix, max_colour_percentile) min_colour_value = numpy.maximum( numpy.nanpercentile(auc_matrix, 100. - max_colour_percentile), 0.5) figure_object, axes_object = _plot_one_value( data_matrix=auc_matrix, grid_metadata_dict=grid_metadata_dict, colour_map_object=score_colour_map_object, min_colour_value=min_colour_value, max_colour_value=max_colour_value, plot_cbar_min_arrow=True, plot_cbar_max_arrow=max_colour_value < 1.) axes_object.set_title('AUC (area under ROC curve)') plotting_utils.label_axes(axes_object=axes_object, label_string='(c)') panel_file_names.append('{0:s}/auc.jpg'.format(output_dir_name)) print('Saving figure to: "{0:s}"...'.format(panel_file_names[-1])) figure_object.savefig(panel_file_names[-1], dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object) # Plot CSI. max_colour_value = numpy.nanpercentile(csi_matrix, max_colour_percentile) min_colour_value = numpy.nanpercentile(csi_matrix, 100. - max_colour_percentile) figure_object, axes_object = _plot_one_value( data_matrix=csi_matrix, grid_metadata_dict=grid_metadata_dict, colour_map_object=score_colour_map_object, min_colour_value=min_colour_value, max_colour_value=max_colour_value, plot_cbar_min_arrow=min_colour_value > 0., plot_cbar_max_arrow=max_colour_value < 1.) axes_object.set_title('CSI (critical success index)') plotting_utils.label_axes(axes_object=axes_object, label_string='(d)') panel_file_names.append('{0:s}/csi.jpg'.format(output_dir_name)) print('Saving figure to: "{0:s}"...'.format(panel_file_names[-1])) figure_object.savefig(panel_file_names[-1], dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object) # Plot POD. max_colour_value = numpy.nanpercentile(pod_matrix, max_colour_percentile) min_colour_value = numpy.nanpercentile(pod_matrix, 100. - max_colour_percentile) figure_object, axes_object = _plot_one_value( data_matrix=pod_matrix, grid_metadata_dict=grid_metadata_dict, colour_map_object=score_colour_map_object, min_colour_value=min_colour_value, max_colour_value=max_colour_value, plot_cbar_min_arrow=min_colour_value > 0., plot_cbar_max_arrow=max_colour_value < 1.) axes_object.set_title('POD (probability of detection)') plotting_utils.label_axes(axes_object=axes_object, label_string='(e)') panel_file_names.append('{0:s}/pod.jpg'.format(output_dir_name)) print('Saving figure to: "{0:s}"...'.format(panel_file_names[-1])) figure_object.savefig(panel_file_names[-1], dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object) # Plot FAR. max_colour_value = numpy.nanpercentile(far_matrix, max_colour_percentile) min_colour_value = numpy.nanpercentile(far_matrix, 100. - max_colour_percentile) figure_object, axes_object = _plot_one_value( data_matrix=far_matrix, grid_metadata_dict=grid_metadata_dict, colour_map_object=score_colour_map_object, min_colour_value=min_colour_value, max_colour_value=max_colour_value, plot_cbar_min_arrow=min_colour_value > 0., plot_cbar_max_arrow=max_colour_value < 1.) axes_object.set_title('FAR (false-alarm ratio)') plotting_utils.label_axes(axes_object=axes_object, label_string='(f)') panel_file_names.append('{0:s}/far.jpg'.format(output_dir_name)) print('Saving figure to: "{0:s}"...'.format(panel_file_names[-1])) figure_object.savefig(panel_file_names[-1], dpi=FIGURE_RESOLUTION_DPI, pad_inches=0, bbox_inches='tight') pyplot.close(figure_object) # Concatenate panels. concat_file_name = '{0:s}/spatially_subset_evaluation.jpg'.format( output_dir_name) print('Concatenating panels to: "{0:s}"...'.format(concat_file_name)) imagemagick_utils.concatenate_images(input_file_names=panel_file_names, output_file_name=concat_file_name, num_panel_rows=NUM_PANEL_ROWS, num_panel_columns=NUM_PANEL_COLUMNS) imagemagick_utils.resize_image(input_file_name=concat_file_name, output_file_name=concat_file_name, output_size_pixels=CONCAT_FIGURE_SIZE_PX)
def _read_one_composite(saliency_file_name, smoothing_radius_grid_cells, monte_carlo_file_name): """Reads saliency map for one composite. E = number of examples M = number of rows in grid N = number of columns in grid H = number of heights in grid F = number of radar fields :param saliency_file_name: Path to input file (will be read by `saliency.read_file`). :param smoothing_radius_grid_cells: Radius for Gaussian smoother, used only for saliency map. :param monte_carlo_file_name: Path to Monte Carlo file (will be read by `_read_monte_carlo_file`). :return: mean_radar_matrix: E-by-M-by-N-by-H-by-F numpy array with mean radar fields. :return: mean_saliency_matrix: E-by-M-by-N-by-H-by-F numpy array with mean saliency fields. :return: significance_matrix: E-by-M-by-N-by-H-by-F numpy array of Boolean flags. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. """ print('Reading saliency maps from: "{0:s}"...'.format(saliency_file_name)) saliency_dict = saliency_maps.read_file(saliency_file_name)[0] mean_radar_matrix = numpy.expand_dims( saliency_dict[saliency_maps.MEAN_PREDICTOR_MATRICES_KEY][0], axis=0) mean_saliency_matrix = numpy.expand_dims( saliency_dict[saliency_maps.MEAN_SALIENCY_MATRICES_KEY][0], axis=0) if smoothing_radius_grid_cells is not None: print(( 'Smoothing saliency maps with Gaussian filter (e-folding radius of ' '{0:.1f} grid cells)...').format(smoothing_radius_grid_cells)) num_fields = mean_radar_matrix.shape[-1] for k in range(num_fields): mean_saliency_matrix[0, ..., k] = ( general_utils.apply_gaussian_filter( input_matrix=mean_saliency_matrix[0, ..., k], e_folding_radius_grid_cells=smoothing_radius_grid_cells)) model_file_name = saliency_dict[saliency_maps.MODEL_FILE_KEY] model_metafile_name = cnn.find_metafile(model_file_name) if monte_carlo_file_name is None: significance_matrix = numpy.full(mean_radar_matrix.shape, False, dtype=bool) else: print('Reading Monte Carlo test from: "{0:s}"...'.format( monte_carlo_file_name)) this_file_handle = open(monte_carlo_file_name, 'rb') monte_carlo_dict = pickle.load(this_file_handle) this_file_handle.close() significance_matrix = numpy.logical_or( monte_carlo_dict[monte_carlo.TRIAL_PMM_MATRICES_KEY][0] < monte_carlo_dict[monte_carlo.MIN_MATRICES_KEY][0], monte_carlo_dict[monte_carlo.TRIAL_PMM_MATRICES_KEY][0] > monte_carlo_dict[monte_carlo.MAX_MATRICES_KEY][0]) significance_matrix = numpy.expand_dims(significance_matrix, axis=0) print('Fraction of significant differences: {0:.4f}'.format( numpy.mean(significance_matrix.astype(float)))) print('Reading CNN metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] good_indices = numpy.array([ numpy.where( training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] == h)[0][0] for h in RADAR_HEIGHTS_M_AGL ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices, :] mean_saliency_matrix = mean_saliency_matrix[..., good_indices, :] significance_matrix = significance_matrix[..., good_indices, :] good_indices = numpy.array([ training_option_dict[trainval_io.RADAR_FIELDS_KEY].index(f) for f in RADAR_FIELD_NAMES ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices] mean_saliency_matrix = mean_saliency_matrix[..., good_indices] significance_matrix = significance_matrix[..., good_indices] training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = RADAR_HEIGHTS_M_AGL training_option_dict[trainval_io.RADAR_FIELDS_KEY] = RADAR_FIELD_NAMES training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = None model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict return (mean_radar_matrix, mean_saliency_matrix, significance_matrix, model_metadata_dict)
def _read_one_composite(gradcam_file_name, smoothing_radius_grid_cells, monte_carlo_file_name, monte_carlo_max_fdr): """Reads class-activation map for one composite. E = number of examples M = number of rows in grid N = number of columns in grid H = number of heights in grid F = number of radar fields :param gradcam_file_name: Path to input file (will be read by `gradcam.read_file`). :param monte_carlo_file_name: Path to Monte Carlo file (will be read by `_read_monte_carlo_file`). :param smoothing_radius_grid_cells: Radius for Gaussian smoother, used only for class-activation map. :param monte_carlo_max_fdr: See documentation at top of file. :return: mean_radar_matrix: E-by-M-by-N-by-H-by-F numpy array with mean radar fields. :return: mean_class_activn_matrix: E-by-M-by-N-by-H numpy array with mean class-activation fields. :return: significance_matrix: E-by-M-by-N-by-H numpy array of Boolean flags. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. """ print('Reading CAMs from: "{0:s}"...'.format(gradcam_file_name)) gradcam_dict = gradcam.read_file(gradcam_file_name)[0] mean_radar_matrix = numpy.expand_dims( gradcam_dict[gradcam.MEAN_PREDICTOR_MATRICES_KEY][0], axis=0) mean_class_activn_matrix = numpy.expand_dims( gradcam_dict[gradcam.MEAN_CAM_MATRICES_KEY][0], axis=0) if smoothing_radius_grid_cells is not None: print( ('Smoothing class-activation maps with Gaussian filter (e-folding ' 'radius of {0:.1f} grid cells)...' ).format(smoothing_radius_grid_cells)) mean_class_activn_matrix[0, ...] = general_utils.apply_gaussian_filter( input_matrix=mean_class_activn_matrix[0, ...], e_folding_radius_grid_cells=smoothing_radius_grid_cells) model_file_name = gradcam_dict[gradcam.MODEL_FILE_KEY] model_metafile_name = cnn.find_metafile(model_file_name) if monte_carlo_file_name is None: significance_matrix = numpy.full(mean_class_activn_matrix.shape, False, dtype=bool) else: print('Reading Monte Carlo test from: "{0:s}"...'.format( monte_carlo_file_name)) this_file_handle = open(monte_carlo_file_name, 'rb') monte_carlo_dict = pickle.load(this_file_handle) this_file_handle.close() p_value_matrix = monte_carlo_dict[monte_carlo.P_VALUE_MATRICES_KEY][0] if monte_carlo_max_fdr is None: significance_matrix = p_value_matrix <= 0.05 else: significance_matrix = monte_carlo.find_sig_grid_points( p_value_matrix=p_value_matrix, max_false_discovery_rate=monte_carlo_max_fdr) significance_matrix = numpy.expand_dims(significance_matrix, axis=0) print('Fraction of significant differences: {0:.4f}'.format( numpy.mean(significance_matrix.astype(float)))) print('Reading CNN metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] good_indices = numpy.array([ numpy.where( training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] == h)[0][0] for h in RADAR_HEIGHTS_M_AGL ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices, :] mean_class_activn_matrix = mean_class_activn_matrix[..., good_indices] significance_matrix = significance_matrix[..., good_indices] good_indices = numpy.array([ training_option_dict[trainval_io.RADAR_FIELDS_KEY].index(f) for f in RADAR_FIELD_NAMES ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices] training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = RADAR_HEIGHTS_M_AGL training_option_dict[trainval_io.RADAR_FIELDS_KEY] = RADAR_FIELD_NAMES training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = None model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict return (mean_radar_matrix, mean_class_activn_matrix, significance_matrix, model_metadata_dict)