def _read_bwo_file(bwo_file_name): """Reads backwards-optimization results from file. :param bwo_file_name: Path to input file (will be read by `backwards_optimization.read_file`). :return: bwo_dictionary: Dictionary returned by `backwards_optimization.read_file`. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. """ print('Reading data from: "{0:s}"...'.format(bwo_file_name)) bwo_dictionary = backwards_opt.read_file(bwo_file_name)[0] mean_before_matrices = [ numpy.expand_dims(a, axis=0) for a in bwo_dictionary[backwards_opt.MEAN_INPUT_MATRICES_KEY] ] mean_after_matrices = [ numpy.expand_dims(a, axis=0) for a in bwo_dictionary[backwards_opt.MEAN_OUTPUT_MATRICES_KEY] ] model_file_name = bwo_dictionary[backwards_opt.MODEL_FILE_KEY] model_metafile_name = cnn.find_metafile(model_file_name) print('Reading CNN metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] good_indices = numpy.array([ numpy.where( training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] == h)[0][0] for h in RADAR_HEIGHTS_M_AGL ], dtype=int) mean_before_matrices[0] = mean_before_matrices[0][..., good_indices, :] mean_after_matrices[0] = mean_after_matrices[0][..., good_indices, :] good_indices = numpy.array([ training_option_dict[trainval_io.RADAR_FIELDS_KEY].index(f) for f in RADAR_FIELD_NAMES ], dtype=int) mean_before_matrices[0] = mean_before_matrices[0][..., good_indices] mean_after_matrices[0] = mean_after_matrices[0][..., good_indices] bwo_dictionary[ backwards_opt.MEAN_INPUT_MATRICES_KEY] = mean_before_matrices bwo_dictionary[ backwards_opt.MEAN_OUTPUT_MATRICES_KEY] = mean_after_matrices training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = RADAR_HEIGHTS_M_AGL training_option_dict[trainval_io.RADAR_FIELDS_KEY] = RADAR_FIELD_NAMES model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict return bwo_dictionary, model_metadata_dict
def _read_one_composite(saliency_file_name, smoothing_radius_grid_cells): """Reads saliency map for one composite. T = number of input tensors for model :param saliency_file_name: Path to input file (will be read by `saliency.read_file`). :param smoothing_radius_grid_cells: Radius for Gaussian smoother, used only for saliency map. :return: mean_predictor_matrices: length-T list of numpy arrays, where the [i]th item has dimensions of the [i]th input tensor for the model. :return: mean_saliency_matrices: Same as `mean_predictor_matrices` but with saliency values. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. """ print('Reading data from: "{0:s}"...'.format(saliency_file_name)) saliency_dict = saliency_maps.read_file(saliency_file_name)[0] mean_predictor_matrices = saliency_dict[MEAN_PREDICTOR_MATRICES_KEY] mean_saliency_matrices = saliency_dict[MEAN_SALIENCY_MATRICES_KEY] num_matrices = len(mean_predictor_matrices) for i in range(num_matrices): mean_predictor_matrices[i] = numpy.expand_dims( mean_predictor_matrices[i], axis=0) mean_saliency_matrices[i] = numpy.expand_dims( mean_saliency_matrices[i], axis=0) model_file_name = saliency_dict[MODEL_FILE_KEY] model_metafile_name = cnn.find_metafile(model_file_name) print('Reading CNN metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY] = False all_refl_heights_m_agl = training_option_dict[ trainval_io.RADAR_HEIGHTS_KEY] good_flags = numpy.array( [h in REFL_HEIGHTS_M_AGL for h in all_refl_heights_m_agl], dtype=bool) good_indices = numpy.where(good_flags)[0] mean_predictor_matrices[0] = (mean_predictor_matrices[0][..., good_indices, :]) mean_saliency_matrices[0] = (mean_saliency_matrices[0][..., good_indices, :]) training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = REFL_HEIGHTS_M_AGL model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict if smoothing_radius_grid_cells is not None: mean_saliency_matrices = _smooth_maps( saliency_matrices=mean_saliency_matrices, smoothing_radius_grid_cells=smoothing_radius_grid_cells) return mean_predictor_matrices, mean_saliency_matrices, model_metadata_dict
def _run(input_file_name, plot_soundings, allow_whitespace, plot_panel_names, add_titles, label_colour_bars, colour_bar_length, output_dir_name): """Plots PMM composite over many examples (storm objects). This is effectively the main method. :param input_file_name: See documentation at top of file. :param plot_soundings: Same. :param allow_whitespace: Same. :param plot_panel_names: Same. :param add_titles: Same. :param label_colour_bars: Same. :param colour_bar_length: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) print('Reading data from: "{0:s}"...'.format(input_file_name)) pickle_file_handle = open(input_file_name, 'rb') input_dict = pickle.load(pickle_file_handle) pickle_file_handle.close() mean_predictor_matrices = input_dict[MEAN_PREDICTOR_MATRICES_KEY] for i in range(len(mean_predictor_matrices)): mean_predictor_matrices[i] = numpy.expand_dims( mean_predictor_matrices[i], axis=0 ) model_file_name = input_dict[MODEL_FILE_KEY] model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0] ) print('Reading metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY][ trainval_io.UPSAMPLE_REFLECTIVITY_KEY ] = False mean_sounding_pressures_pascals = input_dict[MEAN_SOUNDING_PRESSURES_KEY] sounding_pressure_matrix_pascals = numpy.reshape( mean_sounding_pressures_pascals, (1, len(mean_sounding_pressures_pascals)) ) plot_input_examples.plot_examples( list_of_predictor_matrices=mean_predictor_matrices, model_metadata_dict=model_metadata_dict, pmm_flag=True, output_dir_name=output_dir_name, plot_soundings=plot_soundings, sounding_pressure_matrix_pascals=sounding_pressure_matrix_pascals, allow_whitespace=allow_whitespace, plot_panel_names=plot_panel_names, add_titles=add_titles, label_colour_bars=label_colour_bars, colour_bar_length=colour_bar_length)
def _read_composite(pickle_file_name): """Reads PMM composite of examples (storm objects) from Pickle file. T = number of input tensors to model H_s = number of sounding heights :param pickle_file_name: Path to input file. :return: mean_predictor_matrices: length-T of numpy arrays, where the [i]th item has dimensions of the [i]th input tensor to the model. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. :return: mean_sounding_pressures_pa: numpy array (length H_s) of sounding pressures. """ print('Reading data from: "{0:s}"...'.format(pickle_file_name)) file_handle = open(pickle_file_name, 'rb') composite_dict = pickle.load(file_handle) file_handle.close() mean_predictor_matrices = composite_dict[MEAN_PREDICTOR_MATRICES_KEY] mean_sounding_pressures_pa = composite_dict[MEAN_SOUNDING_PRESSURES_KEY] for i in range(len(mean_predictor_matrices)): mean_predictor_matrices[i] = numpy.expand_dims( mean_predictor_matrices[i], axis=0) model_file_name = composite_dict[MODEL_FILE_KEY] model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0]) print('Reading metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY][ trainval_io.UPSAMPLE_REFLECTIVITY_KEY] = False all_radar_heights_m_agl = model_metadata_dict[ cnn.TRAINING_OPTION_DICT_KEY][trainval_io.RADAR_HEIGHTS_KEY] good_flags = numpy.array( [h in RADAR_HEIGHTS_M_AGL for h in all_radar_heights_m_agl], dtype=bool) good_indices = numpy.where(good_flags)[0] mean_predictor_matrices[0] = mean_predictor_matrices[0][..., good_indices, :] model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY][ trainval_io.RADAR_HEIGHTS_KEY] = RADAR_HEIGHTS_M_AGL return (mean_predictor_matrices, model_metadata_dict, mean_sounding_pressures_pa)
def _read_bwo_file(bwo_file_name): """Reads backwards-optimization results from file. :param bwo_file_name: Path to input file (will be read by `backwards_optimization.read_file`). :return: bwo_dictionary: Dictionary returned by `backwards_optimization.read_file`. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. """ print('Reading data from: "{0:s}"...'.format(bwo_file_name)) bwo_dictionary = backwards_opt.read_file(bwo_file_name)[0] mean_before_matrices = [ numpy.expand_dims(a, axis=0) for a in bwo_dictionary[MEAN_INPUT_MATRICES_KEY] ] mean_after_matrices = [ numpy.expand_dims(a, axis=0) for a in bwo_dictionary[MEAN_OUTPUT_MATRICES_KEY] ] model_file_name = bwo_dictionary[MODEL_FILE_KEY] model_metafile_name = cnn.find_metafile(model_file_name) print('Reading CNN metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY] = False all_refl_heights_m_agl = training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] good_flags = numpy.array( [h in REFL_HEIGHTS_M_AGL for h in all_refl_heights_m_agl], dtype=bool ) good_indices = numpy.where(good_flags)[0] mean_before_matrices[0] = mean_before_matrices[0][..., good_indices, :] mean_after_matrices[0] = mean_after_matrices[0][..., good_indices, :] bwo_dictionary[MEAN_INPUT_MATRICES_KEY] = mean_before_matrices bwo_dictionary[MEAN_OUTPUT_MATRICES_KEY] = mean_after_matrices training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = REFL_HEIGHTS_M_AGL model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict return bwo_dictionary, model_metadata_dict
def _run(input_file_name, allow_whitespace, plot_soundings, output_dir_name): """Plots probability-matched mean (PMM) of many examples (storm objects). This is effectively the main method. :param input_file_name: See documentation at top of file. :param plot_soundings: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) print('Reading data from: "{0:s}"...'.format(input_file_name)) pickle_file_handle = open(input_file_name, 'rb') input_dict = pickle.load(pickle_file_handle) pickle_file_handle.close() list_of_mean_input_matrices = input_dict[MEAN_INPUT_MATRICES_KEY] for i in range(len(list_of_mean_input_matrices)): list_of_mean_input_matrices[i] = numpy.expand_dims( list_of_mean_input_matrices[i], axis=0 ) model_file_name = input_dict[MODEL_FILE_KEY] model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0] ) print('Reading metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY][ trainval_io.UPSAMPLE_REFLECTIVITY_KEY ] = False plot_input_examples.plot_examples( list_of_predictor_matrices=list_of_mean_input_matrices, model_metadata_dict=model_metadata_dict, output_dir_name=output_dir_name, plot_soundings=plot_soundings, allow_whitespace=allow_whitespace, pmm_flag=True)
def _train_one_upconvnet(gpu_queue, argument_dict): """Trains one upconvolutional network. :param gpu_queue: GPU queue (instance of `multiprocessing.Manager.Queue`). :param argument_dict: Dictionary of CNN arguments, where each key is an input arg to the script train_upconvnet.py. """ import keras from keras import backend as K import tensorflow from gewittergefahr.deep_learning import cnn from gewittergefahr.deep_learning import upconvnet from gewittergefahr.scripts import train_upconvnet gpu_index = -1 try: # Deal with GPU business. gpu_index = int(gpu_queue.get()) os.environ['CUDA_VISIBLE_DEVICES'] = '{0:d}'.format(gpu_index) session_object = tensorflow.Session( config=tensorflow.ConfigProto( intra_op_parallelism_threads=7, inter_op_parallelism_threads=7, allow_soft_placement=False, log_device_placement=False, gpu_options=tensorflow.GPUOptions(allow_growth=True) ) ) K.set_session(session_object) # Write metadata. upconvnet_metadata_dict = _write_metadata_one_model(argument_dict) # Read trained CNN. cnn_file_name = argument_dict[train_upconvnet.CNN_FILE_ARG_NAME] cnn_metafile_name = cnn.find_metafile(cnn_file_name) print('Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name)) cnn_metadata_dict = cnn.read_model_metadata(cnn_metafile_name) upconvnet_template_file_name = argument_dict[ train_upconvnet.UPCONVNET_FILE_ARG_NAME] with tensorflow.device('/gpu:0'): print('Reading trained CNN from: "{0:s}"...'.format(cnn_file_name)) cnn_model_object = cnn.read_model(cnn_file_name) print('Reading upconvnet architecture from: "{0:s}"...'.format( upconvnet_template_file_name )) upconvnet_model_object = cnn.read_model( upconvnet_template_file_name) upconvnet_model_object.compile( loss=keras.losses.mean_squared_error, optimizer=keras.optimizers.Adam() ) print(SEPARATOR_STRING) upconvnet_model_object.summary() print(SEPARATOR_STRING) # Train upconvnet. print('Training upconvnet on GPU {0:d}...'.format(gpu_index)) print(SEPARATOR_STRING) upconvnet.train_upconvnet( upconvnet_model_object=upconvnet_model_object, output_dir_name=argument_dict[train_upconvnet.OUTPUT_DIR_ARG_NAME], cnn_model_object=cnn_model_object, cnn_metadata_dict=cnn_metadata_dict, cnn_feature_layer_name= upconvnet_metadata_dict[upconvnet.CNN_FEATURE_LAYER_KEY], num_epochs=upconvnet_metadata_dict[upconvnet.NUM_EPOCHS_KEY], num_examples_per_batch= upconvnet_metadata_dict[upconvnet.NUM_EXAMPLES_PER_BATCH_KEY], num_training_batches_per_epoch= upconvnet_metadata_dict[upconvnet.NUM_TRAINING_BATCHES_KEY], training_example_file_names= upconvnet_metadata_dict[upconvnet.TRAINING_FILES_KEY], first_training_time_unix_sec= upconvnet_metadata_dict[upconvnet.FIRST_TRAINING_TIME_KEY], last_training_time_unix_sec= upconvnet_metadata_dict[upconvnet.LAST_TRAINING_TIME_KEY], num_validation_batches_per_epoch= upconvnet_metadata_dict[upconvnet.NUM_VALIDATION_BATCHES_KEY], validation_example_file_names= upconvnet_metadata_dict[upconvnet.VALIDATION_FILES_KEY], first_validation_time_unix_sec= upconvnet_metadata_dict[upconvnet.FIRST_VALIDATION_TIME_KEY], last_validation_time_unix_sec= upconvnet_metadata_dict[upconvnet.LAST_VALIDATION_TIME_KEY] ) session_object.close() del session_object gpu_queue.put(gpu_index) except Exception as this_exception: if gpu_index >= 0: gpu_queue.put(gpu_index) print(traceback.format_exc()) raise this_exception
def _read_one_composite(saliency_file_name, smoothing_radius_grid_cells): """Reads saliency map for one composite. E = number of examples M = number of rows in grid N = number of columns in grid H = number of heights in grid F = number of radar fields :param saliency_file_name: Path to input file (will be read by `saliency.read_file`). :param smoothing_radius_grid_cells: Radius for Gaussian smoother, used only for saliency map. :return: mean_radar_matrix: E-by-M-by-N-by-H-by-F numpy array with mean radar fields. :return: mean_saliency_matrix: E-by-M-by-N-by-H-by-F numpy array with mean saliency fields. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. """ print('Reading data from: "{0:s}"...'.format(saliency_file_name)) saliency_dict = saliency_maps.read_file(saliency_file_name)[0] mean_radar_matrix = numpy.expand_dims( saliency_dict[saliency_maps.MEAN_PREDICTOR_MATRICES_KEY][0], axis=0) mean_saliency_matrix = numpy.expand_dims( saliency_dict[saliency_maps.MEAN_SALIENCY_MATRICES_KEY][0], axis=0) if smoothing_radius_grid_cells is not None: print(( 'Smoothing saliency maps with Gaussian filter (e-folding radius of ' '{0:.1f} grid cells)...').format(smoothing_radius_grid_cells)) num_fields = mean_radar_matrix.shape[-1] for k in range(num_fields): mean_saliency_matrix[0, ..., k] = ( general_utils.apply_gaussian_filter( input_matrix=mean_saliency_matrix[0, ..., k], e_folding_radius_grid_cells=smoothing_radius_grid_cells)) model_file_name = saliency_dict[saliency_maps.MODEL_FILE_KEY] model_metafile_name = cnn.find_metafile(model_file_name) print('Reading CNN metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] good_indices = numpy.array([ numpy.where( training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] == h)[0][0] for h in RADAR_HEIGHTS_M_AGL ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices, :] mean_saliency_matrix = mean_saliency_matrix[..., good_indices, :] good_indices = numpy.array([ training_option_dict[trainval_io.RADAR_FIELDS_KEY].index(f) for f in RADAR_FIELD_NAMES ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices] mean_saliency_matrix = mean_saliency_matrix[..., good_indices] training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = RADAR_HEIGHTS_M_AGL training_option_dict[trainval_io.RADAR_FIELDS_KEY] = RADAR_FIELD_NAMES training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = None model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict return mean_radar_matrix, mean_saliency_matrix, model_metadata_dict
def _run(prediction_file_name, top_example_dir_name, diff_colour_map_name, num_examples, allow_whitespace, plot_panel_names, add_titles, label_colour_bars, colour_bar_length, top_output_dir_name): """Plots one or more radar images and their upconvnet reconstructions. This is effectively the main method. :param prediction_file_name: See documentation at top of file. :param top_example_dir_name: Same. :param diff_colour_map_name: Same. :param num_examples: Same. :param allow_whitespace: Same. :param plot_panel_names: Same. :param add_titles: Same. :param label_colour_bars: Same. :param colour_bar_length: Same. :param top_output_dir_name: Same. """ diff_colour_map_object = pyplot.get_cmap(diff_colour_map_name) # Read data. print('Reading reconstructed radar images from: "{0:s}"...'.format( prediction_file_name )) prediction_dict = upconvnet.read_predictions(prediction_file_name) reconstructed_matrices = prediction_dict[ upconvnet.RECON_IMAGE_MATRICES_KEY] full_storm_id_strings = prediction_dict[upconvnet.FULL_STORM_IDS_KEY] storm_times_unix_sec = prediction_dict[upconvnet.STORM_TIMES_KEY] if 0 < num_examples < len(full_storm_id_strings): reconstructed_matrices = [ a[:num_examples, ...] for a in reconstructed_matrices ] full_storm_id_strings = full_storm_id_strings[:num_examples] storm_times_unix_sec = storm_times_unix_sec[:num_examples] upconvnet_file_name = prediction_dict[upconvnet.UPCONVNET_FILE_KEY] upconvnet_metafile_name = cnn.find_metafile(upconvnet_file_name) print('Reading upconvnet metadata from: "{0:s}"...'.format( upconvnet_metafile_name )) upconvnet_metadata_dict = upconvnet.read_model_metadata( upconvnet_metafile_name) cnn_file_name = upconvnet_metadata_dict[upconvnet.CNN_FILE_KEY] cnn_metafile_name = cnn.find_metafile(cnn_file_name) print('Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name)) cnn_metadata_dict = cnn.read_model_metadata(cnn_metafile_name) training_option_dict = cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] training_option_dict[trainval_io.NORMALIZATION_TYPE_KEY] = None training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = None training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY] = False cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict print(SEPARATOR_STRING) example_dict = testing_io.read_predictors_specific_examples( top_example_dir_name=top_example_dir_name, desired_full_id_strings=full_storm_id_strings, desired_times_unix_sec=storm_times_unix_sec, option_dict=training_option_dict, layer_operation_dicts=cnn_metadata_dict[cnn.LAYER_OPERATIONS_KEY] ) print(SEPARATOR_STRING) plot_examples.plot_examples( list_of_predictor_matrices=example_dict[testing_io.INPUT_MATRICES_KEY], model_metadata_dict=cnn_metadata_dict, output_dir_name='{0:s}/actual_images'.format(top_output_dir_name), pmm_flag=False, plot_soundings=False, plot_radar_diffs=False, allow_whitespace=allow_whitespace, plot_panel_names=plot_panel_names, add_titles=add_titles, label_colour_bars=label_colour_bars, colour_bar_length=colour_bar_length, full_storm_id_strings=full_storm_id_strings, storm_times_unix_sec=storm_times_unix_sec) print(SEPARATOR_STRING) plot_examples.plot_examples( list_of_predictor_matrices=reconstructed_matrices, model_metadata_dict=cnn_metadata_dict, output_dir_name= '{0:s}/reconstructed_images'.format(top_output_dir_name), pmm_flag=False, plot_soundings=False, plot_radar_diffs=False, allow_whitespace=allow_whitespace, plot_panel_names=plot_panel_names, add_titles=add_titles, label_colour_bars=label_colour_bars, colour_bar_length=colour_bar_length, full_storm_id_strings=full_storm_id_strings, storm_times_unix_sec=storm_times_unix_sec) print(SEPARATOR_STRING) difference_matrices = [ r - a for r, a in zip( reconstructed_matrices, example_dict[testing_io.INPUT_MATRICES_KEY] ) ] plot_examples.plot_examples( list_of_predictor_matrices=difference_matrices, model_metadata_dict=cnn_metadata_dict, output_dir_name='{0:s}/differences'.format(top_output_dir_name), pmm_flag=False, plot_soundings=False, plot_radar_diffs=True, diff_colour_map_object=diff_colour_map_object, max_diff_percentile=99., allow_whitespace=allow_whitespace, plot_panel_names=plot_panel_names, add_titles=add_titles, label_colour_bars=label_colour_bars, colour_bar_length=colour_bar_length, full_storm_id_strings=full_storm_id_strings, storm_times_unix_sec=storm_times_unix_sec)
def _run(input_file_name, plot_soundings, allow_whitespace, plot_significance, colour_map_name, max_colour_percentile, output_dir_name): """Plots saliency maps for a CNN (convolutional neural network). This is effectively the main method. :param input_file_name: See documentation at top of file. :param plot_soundings: Same. :param allow_whitespace: Same. :param plot_significance: Same. :param colour_map_name: Same. :param max_colour_percentile: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) error_checking.assert_is_geq(max_colour_percentile, 0.) error_checking.assert_is_leq(max_colour_percentile, 100.) colour_map_object = pyplot.cm.get_cmap(colour_map_name) print('Reading data from: "{0:s}"...'.format(input_file_name)) try: saliency_dict = saliency_maps.read_standard_file(input_file_name) list_of_input_matrices = saliency_dict.pop( saliency_maps.INPUT_MATRICES_KEY) list_of_saliency_matrices = saliency_dict.pop( saliency_maps.SALIENCY_MATRICES_KEY) full_storm_id_strings = saliency_dict[saliency_maps.FULL_IDS_KEY] storm_times_unix_sec = saliency_dict[saliency_maps.STORM_TIMES_KEY] except ValueError: saliency_dict = saliency_maps.read_pmm_file(input_file_name) list_of_input_matrices = saliency_dict.pop( saliency_maps.MEAN_INPUT_MATRICES_KEY) list_of_saliency_matrices = saliency_dict.pop( saliency_maps.MEAN_SALIENCY_MATRICES_KEY) for i in range(len(list_of_input_matrices)): list_of_input_matrices[i] = numpy.expand_dims( list_of_input_matrices[i], axis=0 ) list_of_saliency_matrices[i] = numpy.expand_dims( list_of_saliency_matrices[i], axis=0 ) full_storm_id_strings = [None] storm_times_unix_sec = [None] pmm_flag = ( full_storm_id_strings[0] is None and storm_times_unix_sec[0] is None ) num_examples = list_of_input_matrices[0].shape[0] max_colour_value_by_example = numpy.full(num_examples, numpy.nan) for i in range(num_examples): these_saliency_values = numpy.concatenate( [numpy.ravel(s[i, ...]) for s in list_of_saliency_matrices] ) max_colour_value_by_example[i] = numpy.percentile( numpy.absolute(these_saliency_values), max_colour_percentile ) model_file_name = saliency_dict[saliency_maps.MODEL_FILE_KEY] model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0] ) print('Reading metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) print(SEPARATOR_STRING) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] has_soundings = ( training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is not None ) num_radar_matrices = len(list_of_input_matrices) - int(has_soundings) monte_carlo_dict = ( saliency_dict[saliency_maps.MONTE_CARLO_DICT_KEY] if plot_significance and saliency_maps.MONTE_CARLO_DICT_KEY in saliency_dict else None ) for i in range(num_examples): if has_soundings and plot_soundings: _plot_sounding_saliency( saliency_matrix=list_of_saliency_matrices[-1][i, ...], colour_map_object=colour_map_object, max_colour_value=max_colour_value_by_example[i], sounding_matrix=list_of_input_matrices[-1][i, ...], saliency_dict=saliency_dict, model_metadata_dict=model_metadata_dict, output_dir_name=output_dir_name, pmm_flag=pmm_flag, example_index=i) this_handle_dict = plot_input_examples.plot_one_example( list_of_predictor_matrices=list_of_input_matrices, model_metadata_dict=model_metadata_dict, plot_sounding=False, allow_whitespace=allow_whitespace, pmm_flag=pmm_flag, example_index=i, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i] ) these_figure_objects = this_handle_dict[ plot_input_examples.RADAR_FIGURES_KEY] these_axes_object_matrices = this_handle_dict[ plot_input_examples.RADAR_AXES_KEY] for j in range(num_radar_matrices): if monte_carlo_dict is None: this_significance_matrix = None else: this_significance_matrix = numpy.logical_or( monte_carlo_dict[ monte_carlo.TRIAL_PMM_MATRICES_KEY][j][i, ...] < monte_carlo_dict[monte_carlo.MIN_MATRICES_KEY][j][i, ...], monte_carlo_dict[ monte_carlo.TRIAL_PMM_MATRICES_KEY][j][i, ...] > monte_carlo_dict[monte_carlo.MAX_MATRICES_KEY][j][i, ...] ) this_num_spatial_dim = len(list_of_input_matrices[j].shape) - 2 if this_num_spatial_dim == 3: _plot_3d_radar_saliency( saliency_matrix=list_of_saliency_matrices[j][i, ...], colour_map_object=colour_map_object, max_colour_value=max_colour_value_by_example[i], figure_objects=these_figure_objects, axes_object_matrices=these_axes_object_matrices, model_metadata_dict=model_metadata_dict, output_dir_name=output_dir_name, significance_matrix=this_significance_matrix, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i] ) else: _plot_2d_radar_saliency( saliency_matrix=list_of_saliency_matrices[j][i, ...], colour_map_object=colour_map_object, max_colour_value=max_colour_value_by_example[i], figure_objects=these_figure_objects, axes_object_matrices=these_axes_object_matrices, model_metadata_dict=model_metadata_dict, output_dir_name=output_dir_name, significance_matrix=this_significance_matrix, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i] )
def _run(model_file_name, top_example_dir_name, storm_metafile_name, num_examples, output_file_name): """Creates dummy saliency map for each storm object. This is effectively the main method. :param model_file_name: See documentation at top of file. :param top_example_dir_name: Same. :param storm_metafile_name: Same. :param num_examples: Same. :param output_file_name: Same. """ file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0]) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None print( 'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name)) full_storm_id_strings, storm_times_unix_sec = ( tracking_io.read_ids_and_times(storm_metafile_name)) print(SEPARATOR_STRING) if 0 < num_examples < len(full_storm_id_strings): full_storm_id_strings = full_storm_id_strings[:num_examples] storm_times_unix_sec = storm_times_unix_sec[:num_examples] example_dict = testing_io.read_predictors_specific_examples( top_example_dir_name=top_example_dir_name, desired_full_id_strings=full_storm_id_strings, desired_times_unix_sec=storm_times_unix_sec, option_dict=training_option_dict, layer_operation_dicts=model_metadata_dict[cnn.LAYER_OPERATIONS_KEY]) print(SEPARATOR_STRING) predictor_matrices = example_dict[testing_io.INPUT_MATRICES_KEY] sounding_pressure_matrix_pa = ( example_dict[testing_io.SOUNDING_PRESSURES_KEY]) radar_matrix = predictor_matrices[0] num_examples = radar_matrix.shape[0] num_channels = radar_matrix.shape[-1] num_spatial_dim = len(radar_matrix.shape) - 2 if num_spatial_dim == 2: kernel_matrix = numpy.expand_dims(EDGE_DETECTOR_MATRIX_2D, axis=-1) else: kernel_matrix = numpy.expand_dims(EDGE_DETECTOR_MATRIX_3D, axis=-1) kernel_matrix = numpy.repeat(kernel_matrix, num_channels, axis=-1) kernel_matrix = numpy.expand_dims(kernel_matrix, axis=-1) kernel_matrix = numpy.repeat(kernel_matrix, num_channels, axis=-1) radar_saliency_matrix = numpy.full(radar_matrix.shape, numpy.nan) for i in range(num_examples): if numpy.mod(i, 10) == 0: print(( 'Have created dummy saliency map for {0:d} of {1:d} examples...' ).format(i, num_examples)) if num_spatial_dim == 2: this_saliency_matrix = standalone_utils.do_2d_convolution( feature_matrix=radar_matrix[i, ...], kernel_matrix=kernel_matrix, pad_edges=True, stride_length_px=1) else: this_saliency_matrix = standalone_utils.do_3d_convolution( feature_matrix=radar_matrix[i, ...], kernel_matrix=kernel_matrix, pad_edges=True, stride_length_px=1) radar_saliency_matrix[i, ...] = this_saliency_matrix[0, ...] print('Have created dummy saliency map for all {0:d} examples!'.format( num_examples)) print(SEPARATOR_STRING) saliency_matrices = [ radar_saliency_matrix if k == 0 else predictor_matrices[k] for k in range(len(predictor_matrices)) ] saliency_matrices = trainval_io.separate_shear_and_reflectivity( list_of_input_matrices=saliency_matrices, training_option_dict=training_option_dict) denorm_predictor_matrices = trainval_io.separate_shear_and_reflectivity( list_of_input_matrices=copy.deepcopy(predictor_matrices), training_option_dict=training_option_dict) print('Denormalizing model inputs...') denorm_predictor_matrices = model_interpretation.denormalize_data( list_of_input_matrices=denorm_predictor_matrices, model_metadata_dict=model_metadata_dict) print('Writing saliency maps to file: "{0:s}"...'.format(output_file_name)) saliency_metadata_dict = saliency_maps.check_metadata( component_type_string=model_interpretation.CLASS_COMPONENT_TYPE_STRING, target_class=1) saliency_maps.write_standard_file( pickle_file_name=output_file_name, denorm_predictor_matrices=denorm_predictor_matrices, saliency_matrices=saliency_matrices, full_storm_id_strings=full_storm_id_strings, storm_times_unix_sec=storm_times_unix_sec, model_file_name=model_file_name, metadata_dict=saliency_metadata_dict, sounding_pressure_matrix_pa=sounding_pressure_matrix_pa)
def _read_one_composite(saliency_file_name, smoothing_radius_grid_cells, monte_carlo_file_name): """Reads saliency map for one composite. T = number of model-input tensors with radar data :param saliency_file_name: Path to input file (will be read by `saliency.read_file`). :param smoothing_radius_grid_cells: Radius for Gaussian smoother, used only for saliency map. :param monte_carlo_file_name: Path to Monte Carlo file (will be read by `_read_monte_carlo_file`). :return: mean_radar_matrices: length-T list of numpy arrays with mean predictor values. The [i]th array has the same dimensions as the [i]th input tensor to the model. :return: mean_saliency_matrices: Same but with saliency values. :return: significance_matrices: Same but with Boolean significance flags. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. """ print('Reading data from: "{0:s}"...'.format(saliency_file_name)) saliency_dict = saliency_maps.read_file(saliency_file_name)[0] mean_radar_matrices = saliency_dict[MEAN_PREDICTOR_MATRICES_KEY] mean_saliency_matrices = saliency_dict[MEAN_SALIENCY_MATRICES_KEY] num_matrices = len(mean_radar_matrices) for i in range(num_matrices): mean_radar_matrices[i] = numpy.expand_dims(mean_radar_matrices[i], axis=0) mean_saliency_matrices[i] = numpy.expand_dims( mean_saliency_matrices[i], axis=0) model_file_name = saliency_dict[MODEL_FILE_KEY] model_metafile_name = cnn.find_metafile(model_file_name) print('Reading CNN metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY] = False has_soundings = (training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is not None) if has_soundings: mean_radar_matrices = mean_radar_matrices[:-1] mean_saliency_matrices = mean_saliency_matrices[:-1] if smoothing_radius_grid_cells is not None: mean_saliency_matrices = _smooth_maps( saliency_matrices=mean_saliency_matrices, smoothing_radius_grid_cells=smoothing_radius_grid_cells) num_matrices = len(mean_radar_matrices) significance_matrices = [None] * num_matrices if monte_carlo_file_name is None: for i in range(num_matrices): significance_matrices[i] = numpy.full(mean_radar_matrices[i].shape, False, dtype=bool) else: print('Reading Monte Carlo test from: "{0:s}"...'.format( monte_carlo_file_name)) this_file_handle = open(monte_carlo_file_name, 'rb') monte_carlo_dict = pickle.load(this_file_handle) this_file_handle.close() for i in range(num_matrices): significance_matrices[i] = ( monte_carlo_dict[monte_carlo.P_VALUE_MATRICES_KEY][i] <= 0.05) significance_matrices[i] = numpy.expand_dims( significance_matrices[i], axis=0) all_sig_flags = numpy.concatenate( [numpy.ravel(a) for a in significance_matrices]) print('Fraction of significant differences: {0:.4f}'.format( numpy.mean(all_sig_flags.astype(float)))) all_refl_heights_m_agl = training_option_dict[ trainval_io.RADAR_HEIGHTS_KEY] good_flags = numpy.array( [h in REFL_HEIGHTS_M_AGL for h in all_refl_heights_m_agl], dtype=bool) good_indices = numpy.where(good_flags)[0] mean_radar_matrices[0] = (mean_radar_matrices[0][..., good_indices, :]) mean_saliency_matrices[0] = (mean_saliency_matrices[0][..., good_indices, :]) significance_matrices[0] = (significance_matrices[0][..., good_indices, :]) training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = REFL_HEIGHTS_M_AGL training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = None model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict return (mean_radar_matrices, mean_saliency_matrices, significance_matrices, model_metadata_dict)
def _run(input_file_name, cam_colour_map_name, max_colour_prctile_for_cam, top_output_dir_name): """Plots Grad-CAM output (class-activation maps). This is effectively the main method. :param input_file_name: See documentation at top of file. :param cam_colour_map_name: Same. :param max_colour_prctile_for_cam: Same. :param top_output_dir_name: Same. :raises: TypeError: if input file contains class-activation maps for soundings. """ main_gradcam_dir_name = '{0:s}/main_gradcam'.format(top_output_dir_name) guided_gradcam_dir_name = '{0:s}/guided_gradcam'.format(top_output_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=main_gradcam_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=guided_gradcam_dir_name) # Check input args. error_checking.assert_is_geq(max_colour_prctile_for_cam, 0.) error_checking.assert_is_leq(max_colour_prctile_for_cam, 100.) cam_colour_map_object = pyplot.cm.get_cmap(cam_colour_map_name) print 'Reading data from: "{0:s}"...'.format(input_file_name) try: gradcam_dict = gradcam.read_standard_file(input_file_name) list_of_input_matrices = gradcam_dict.pop(gradcam.INPUT_MATRICES_KEY) class_activation_matrix = gradcam_dict.pop( gradcam.CLASS_ACTIVATIONS_KEY) ggradcam_output_matrix = gradcam_dict.pop(gradcam.GUIDED_GRADCAM_KEY) gradcam_metadata_dict = gradcam_dict storm_ids = gradcam_metadata_dict[gradcam.STORM_IDS_KEY] storm_times_unix_sec = gradcam_metadata_dict[gradcam.STORM_TIMES_KEY] except ValueError: gradcam_dict = gradcam.read_pmm_file(input_file_name) list_of_input_matrices = gradcam_dict[gradcam.MEAN_INPUT_MATRICES_KEY] class_activation_matrix = gradcam_dict[ gradcam.MEAN_CLASS_ACTIVATIONS_KEY] ggradcam_output_matrix = gradcam_dict[gradcam.MEAN_GUIDED_GRADCAM_KEY] for i in range(len(list_of_input_matrices)): list_of_input_matrices[i] = numpy.expand_dims( list_of_input_matrices[i], axis=0) class_activation_matrix = numpy.expand_dims( class_activation_matrix, axis=0) ggradcam_output_matrix = numpy.expand_dims( ggradcam_output_matrix, axis=0) orig_gradcam_file_name = gradcam_dict[gradcam.STANDARD_FILE_NAME_KEY] print 'Reading metadata from: "{0:s}"...'.format(orig_gradcam_file_name) orig_gradcam_dict = gradcam.read_standard_file(orig_gradcam_file_name) orig_gradcam_dict.pop(gradcam.INPUT_MATRICES_KEY) orig_gradcam_dict.pop(gradcam.CLASS_ACTIVATIONS_KEY) orig_gradcam_dict.pop(gradcam.GUIDED_GRADCAM_KEY) gradcam_metadata_dict = orig_gradcam_dict storm_ids = None storm_times_unix_sec = None num_spatial_dimensions = len(class_activation_matrix.shape) - 1 if num_spatial_dimensions == 1: raise TypeError('This script is not yet equipped to plot ' 'class-activation maps for soundings.') # Read metadata for CNN. model_file_name = gradcam_metadata_dict[gradcam.MODEL_FILE_NAME_KEY] model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0] ) print 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) print SEPARATOR_STRING # Do plotting. if num_spatial_dimensions == 3: _plot_3d_radar_cams( radar_matrix=list_of_input_matrices[0], class_activation_matrix=class_activation_matrix, model_metadata_dict=model_metadata_dict, cam_colour_map_object=cam_colour_map_object, max_colour_prctile_for_cam=max_colour_prctile_for_cam, output_dir_name=main_gradcam_dir_name, storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec) print SEPARATOR_STRING _plot_3d_radar_cams( radar_matrix=list_of_input_matrices[0], ggradcam_output_matrix=ggradcam_output_matrix, model_metadata_dict=model_metadata_dict, cam_colour_map_object=cam_colour_map_object, max_colour_prctile_for_cam=max_colour_prctile_for_cam, output_dir_name=guided_gradcam_dir_name, storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec) else: if len(list_of_input_matrices) == 3: radar_matrix = list_of_input_matrices[1] else: radar_matrix = list_of_input_matrices[0] _plot_2d_radar_cams( radar_matrix=radar_matrix, class_activation_matrix=class_activation_matrix, model_metadata_dict=model_metadata_dict, cam_colour_map_object=cam_colour_map_object, max_colour_prctile_for_cam=max_colour_prctile_for_cam, output_dir_name=main_gradcam_dir_name, storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec) print SEPARATOR_STRING _plot_2d_radar_cams( radar_matrix=radar_matrix, ggradcam_output_matrix=ggradcam_output_matrix, model_metadata_dict=model_metadata_dict, cam_colour_map_object=cam_colour_map_object, max_colour_prctile_for_cam=max_colour_prctile_for_cam, output_dir_name=guided_gradcam_dir_name, storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec)
def _read_one_composite(saliency_file_name, smoothing_radius_grid_cells, monte_carlo_file_name): """Reads saliency map for one composite. E = number of examples M = number of rows in grid N = number of columns in grid H = number of heights in grid F = number of radar fields :param saliency_file_name: Path to input file (will be read by `saliency.read_file`). :param smoothing_radius_grid_cells: Radius for Gaussian smoother, used only for saliency map. :param monte_carlo_file_name: Path to Monte Carlo file (will be read by `_read_monte_carlo_file`). :return: mean_radar_matrix: E-by-M-by-N-by-H-by-F numpy array with mean radar fields. :return: mean_saliency_matrix: E-by-M-by-N-by-H-by-F numpy array with mean saliency fields. :return: significance_matrix: E-by-M-by-N-by-H-by-F numpy array of Boolean flags. :return: model_metadata_dict: Dictionary returned by `cnn.read_model_metadata`. """ print('Reading saliency maps from: "{0:s}"...'.format(saliency_file_name)) saliency_dict = saliency_maps.read_file(saliency_file_name)[0] mean_radar_matrix = numpy.expand_dims( saliency_dict[saliency_maps.MEAN_PREDICTOR_MATRICES_KEY][0], axis=0) mean_saliency_matrix = numpy.expand_dims( saliency_dict[saliency_maps.MEAN_SALIENCY_MATRICES_KEY][0], axis=0) if smoothing_radius_grid_cells is not None: print(( 'Smoothing saliency maps with Gaussian filter (e-folding radius of ' '{0:.1f} grid cells)...').format(smoothing_radius_grid_cells)) num_fields = mean_radar_matrix.shape[-1] for k in range(num_fields): mean_saliency_matrix[0, ..., k] = ( general_utils.apply_gaussian_filter( input_matrix=mean_saliency_matrix[0, ..., k], e_folding_radius_grid_cells=smoothing_radius_grid_cells)) model_file_name = saliency_dict[saliency_maps.MODEL_FILE_KEY] model_metafile_name = cnn.find_metafile(model_file_name) if monte_carlo_file_name is None: significance_matrix = numpy.full(mean_radar_matrix.shape, False, dtype=bool) else: print('Reading Monte Carlo test from: "{0:s}"...'.format( monte_carlo_file_name)) this_file_handle = open(monte_carlo_file_name, 'rb') monte_carlo_dict = pickle.load(this_file_handle) this_file_handle.close() significance_matrix = numpy.logical_or( monte_carlo_dict[monte_carlo.TRIAL_PMM_MATRICES_KEY][0] < monte_carlo_dict[monte_carlo.MIN_MATRICES_KEY][0], monte_carlo_dict[monte_carlo.TRIAL_PMM_MATRICES_KEY][0] > monte_carlo_dict[monte_carlo.MAX_MATRICES_KEY][0]) significance_matrix = numpy.expand_dims(significance_matrix, axis=0) print('Fraction of significant differences: {0:.4f}'.format( numpy.mean(significance_matrix.astype(float)))) print('Reading CNN metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] good_indices = numpy.array([ numpy.where( training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] == h)[0][0] for h in RADAR_HEIGHTS_M_AGL ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices, :] mean_saliency_matrix = mean_saliency_matrix[..., good_indices, :] significance_matrix = significance_matrix[..., good_indices, :] good_indices = numpy.array([ training_option_dict[trainval_io.RADAR_FIELDS_KEY].index(f) for f in RADAR_FIELD_NAMES ], dtype=int) mean_radar_matrix = mean_radar_matrix[..., good_indices] mean_saliency_matrix = mean_saliency_matrix[..., good_indices] significance_matrix = significance_matrix[..., good_indices] training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = RADAR_HEIGHTS_M_AGL training_option_dict[trainval_io.RADAR_FIELDS_KEY] = RADAR_FIELD_NAMES training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = None model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict return (mean_radar_matrix, mean_saliency_matrix, significance_matrix, model_metadata_dict)
def _run(model_file_name, component_type_string, target_class, layer_name, neuron_indices_flattened, channel_indices, top_example_dir_name, first_spc_date_string, last_spc_date_string, output_file_name): """Creates activation maps for one class, neuron, or channel of a CNN. This is effectively the main method. :param model_file_name: See documentation at top of file. :param component_type_string: Same. :param target_class: Same. :param layer_name: Same. :param neuron_indices_flattened: Same. :param channel_indices: Same. :param top_example_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param output_file_name: Same. """ # Check input args. file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) model_interpretation.check_component_type(component_type_string) if component_type_string == CHANNEL_COMPONENT_TYPE_STRING: error_checking.assert_is_geq_numpy_array(channel_indices, 0) if component_type_string == NEURON_COMPONENT_TYPE_STRING: neuron_indices_flattened = neuron_indices_flattened.astype(float) neuron_indices_flattened[neuron_indices_flattened < 0] = numpy.nan neuron_indices_2d_list = general_utils.split_array_by_nan( neuron_indices_flattened) neuron_index_matrix = numpy.array(neuron_indices_2d_list, dtype=int) else: neuron_index_matrix = None # Read model and metadata. print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = cnn.read_model(model_file_name) metadata_file_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0]) print('Reading metadata from: "{0:s}"...'.format(metadata_file_name)) model_metadata_dict = cnn.read_model_metadata(metadata_file_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] # Create generator. example_file_names = input_examples.find_many_example_files( top_directory_name=top_example_dir_name, shuffled=False, first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string, raise_error_if_any_missing=False) training_option_dict[trainval_io.SAMPLING_FRACTIONS_KEY] = None training_option_dict[trainval_io.EXAMPLE_FILES_KEY] = example_file_names training_option_dict[trainval_io.FIRST_STORM_TIME_KEY] = ( time_conversion.get_start_of_spc_date(first_spc_date_string)) training_option_dict[trainval_io.LAST_STORM_TIME_KEY] = ( time_conversion.get_end_of_spc_date(last_spc_date_string)) if model_metadata_dict[cnn.LAYER_OPERATIONS_KEY] is not None: generator_object = testing_io.gridrad_generator_2d_reduced( option_dict=training_option_dict, list_of_operation_dicts=model_metadata_dict[ cnn.LAYER_OPERATIONS_KEY], num_examples_total=LARGE_INTEGER) elif model_metadata_dict[cnn.CONV_2D3D_KEY]: generator_object = testing_io.myrorss_generator_2d3d( option_dict=training_option_dict, num_examples_total=LARGE_INTEGER) else: generator_object = testing_io.generator_2d_or_3d( option_dict=training_option_dict, num_examples_total=LARGE_INTEGER) # Compute activation for each example (storm object) and model component. full_id_strings = [] storm_times_unix_sec = numpy.array([], dtype=int) activation_matrix = None print(SEPARATOR_STRING) for _ in range(len(example_file_names)): try: this_storm_object_dict = next(generator_object) except StopIteration: break this_list_of_input_matrices = this_storm_object_dict[ testing_io.INPUT_MATRICES_KEY] these_id_strings = this_storm_object_dict[testing_io.FULL_IDS_KEY] these_times_unix_sec = this_storm_object_dict[ testing_io.STORM_TIMES_KEY] full_id_strings += these_id_strings storm_times_unix_sec = numpy.concatenate( (storm_times_unix_sec, these_times_unix_sec)) if component_type_string == CLASS_COMPONENT_TYPE_STRING: print('Computing activations for target class {0:d}...'.format( target_class)) this_activation_matrix = ( model_activation.get_class_activation_for_examples( model_object=model_object, target_class=target_class, list_of_input_matrices=this_list_of_input_matrices)) this_activation_matrix = numpy.reshape( this_activation_matrix, (len(this_activation_matrix), 1)) elif component_type_string == NEURON_COMPONENT_TYPE_STRING: this_activation_matrix = None for j in range(neuron_index_matrix.shape[0]): print(( 'Computing activations for neuron {0:s} in layer "{1:s}"...' ).format(str(neuron_index_matrix[j, :]), layer_name)) these_activations = ( model_activation.get_neuron_activation_for_examples( model_object=model_object, layer_name=layer_name, neuron_indices=neuron_index_matrix[j, :], list_of_input_matrices=this_list_of_input_matrices)) these_activations = numpy.reshape(these_activations, (len(these_activations), 1)) if this_activation_matrix is None: this_activation_matrix = these_activations + 0. else: this_activation_matrix = numpy.concatenate( (this_activation_matrix, these_activations), axis=1) else: this_activation_matrix = None for this_channel_index in channel_indices: print(('Computing activations for channel {0:d} in layer ' '"{1:s}"...').format(this_channel_index, layer_name)) these_activations = ( model_activation.get_channel_activation_for_examples( model_object=model_object, layer_name=layer_name, channel_index=this_channel_index, list_of_input_matrices=this_list_of_input_matrices, stat_function_for_neuron_activations=K.max)) these_activations = numpy.reshape(these_activations, (len(these_activations), 1)) if this_activation_matrix is None: this_activation_matrix = these_activations + 0. else: this_activation_matrix = numpy.concatenate( (this_activation_matrix, these_activations), axis=1) if activation_matrix is None: activation_matrix = this_activation_matrix + 0. else: activation_matrix = numpy.concatenate( (activation_matrix, this_activation_matrix), axis=0) print(SEPARATOR_STRING) print('Writing activations to file: "{0:s}"...'.format(output_file_name)) model_activation.write_file(pickle_file_name=output_file_name, activation_matrix=activation_matrix, full_id_strings=full_id_strings, storm_times_unix_sec=storm_times_unix_sec, model_file_name=model_file_name, component_type_string=component_type_string, target_class=target_class, layer_name=layer_name, neuron_index_matrix=neuron_index_matrix, channel_indices=channel_indices)
def _run(upconvnet_file_name, storm_metafile_name, num_examples, top_example_dir_name, top_output_dir_name): """Plots upconvnet reconstructions of many examples (storm objects). This is effectively the main method. :param upconvnet_file_name: See documentation at top of file. :param storm_metafile_name: Same. :param num_examples: Same. :param top_example_dir_name: Same. :param top_output_dir_name: Same. """ print 'Reading trained upconvnet from: "{0:s}"...'.format( upconvnet_file_name) upconvnet_model_object = cnn.read_model(upconvnet_file_name) upconvnet_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(upconvnet_file_name)[0] ) print 'Reading upconvnet metadata from: "{0:s}"...'.format( upconvnet_metafile_name) upconvnet_metadata_dict = upconvnet.read_model_metadata( upconvnet_metafile_name) cnn_file_name = upconvnet_metadata_dict[upconvnet.CNN_FILE_KEY] print 'Reading trained CNN from: "{0:s}"...'.format(cnn_file_name) cnn_model_object = cnn.read_model(cnn_file_name) cnn_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(cnn_file_name)[0] ) print 'Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name) cnn_metadata_dict = cnn.read_model_metadata(cnn_metafile_name) training_option_dict = cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] print 'Reading storm IDs and times from: "{0:s}"...'.format( storm_metafile_name) storm_ids, storm_times_unix_sec = tracking_io.read_ids_and_times( storm_metafile_name) if 0 < num_examples < len(storm_ids): storm_ids = storm_ids[:num_examples] storm_times_unix_sec = storm_times_unix_sec[:num_examples] print SEPARATOR_STRING list_of_predictor_matrices = testing_io.read_specific_examples( desired_storm_ids=storm_ids, desired_times_unix_sec=storm_times_unix_sec, option_dict=training_option_dict, top_example_dir_name=top_example_dir_name, list_of_layer_operation_dicts=cnn_metadata_dict[ cnn.LAYER_OPERATIONS_KEY] )[0] print SEPARATOR_STRING actual_radar_matrix = list_of_predictor_matrices[0] have_soundings = training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] if have_soundings: sounding_matrix = list_of_predictor_matrices[-1] else: sounding_matrix = None feature_matrix = cnn.apply_2d_or_3d_cnn( model_object=cnn_model_object, radar_image_matrix=actual_radar_matrix, sounding_matrix=sounding_matrix, verbose=True, return_features=True, feature_layer_name=upconvnet_metadata_dict[ upconvnet.CNN_FEATURE_LAYER_KEY] ) print '\n' reconstructed_radar_matrix = upconvnet.apply_upconvnet( model_object=upconvnet_model_object, feature_matrix=feature_matrix, verbose=True) print '\n' print 'Denormalizing actual and reconstructed radar images...' cnn_metadata_dict[ cnn.TRAINING_OPTION_DICT_KEY][trainval_io.SOUNDING_FIELDS_KEY] = None actual_radar_matrix = model_interpretation.denormalize_data( list_of_input_matrices=[actual_radar_matrix], model_metadata_dict=cnn_metadata_dict )[0] reconstructed_radar_matrix = model_interpretation.denormalize_data( list_of_input_matrices=[reconstructed_radar_matrix], model_metadata_dict=cnn_metadata_dict )[0] print SEPARATOR_STRING actual_output_dir_name = '{0:s}/actual_images'.format(top_output_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=actual_output_dir_name) # TODO(thunderhoser): Calling a method in another script is hacky. If this # method is going to be reused, should be in a module. plot_input_examples.plot_examples( list_of_predictor_matrices=[actual_radar_matrix], storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec, model_metadata_dict=cnn_metadata_dict, output_dir_name=actual_output_dir_name) print SEPARATOR_STRING reconstructed_output_dir_name = '{0:s}/reconstructed_images'.format( top_output_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=reconstructed_output_dir_name) plot_input_examples.plot_examples( list_of_predictor_matrices=[reconstructed_radar_matrix], storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec, model_metadata_dict=cnn_metadata_dict, output_dir_name=reconstructed_output_dir_name)
def _run(model_file_name, example_file_name, first_time_string, last_time_string, top_output_dir_name): """Applies CNN to one example file. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_file_name: Same. :param first_time_string: Same. :param last_time_string: Same. :param top_output_dir_name: Same. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = cnn.read_model(model_file_name) model_directory_name, _ = os.path.split(model_file_name) model_metafile_name = '{0:s}/model_metadata.p'.format(model_directory_name) print('Reading metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] first_time_unix_sec = time_conversion.string_to_unix_sec( first_time_string, INPUT_TIME_FORMAT) last_time_unix_sec = time_conversion.string_to_unix_sec( last_time_string, INPUT_TIME_FORMAT) training_option_dict[trainval_io.SAMPLING_FRACTIONS_KEY] = None training_option_dict[trainval_io.EXAMPLE_FILES_KEY] = [example_file_name] training_option_dict[ trainval_io.FIRST_STORM_TIME_KEY] = first_time_unix_sec training_option_dict[trainval_io.LAST_STORM_TIME_KEY] = last_time_unix_sec if model_metadata_dict[cnn.LAYER_OPERATIONS_KEY] is not None: generator_object = testing_io.gridrad_generator_2d_reduced( option_dict=training_option_dict, list_of_operation_dicts=model_metadata_dict[ cnn.LAYER_OPERATIONS_KEY], num_examples_total=LARGE_INTEGER) elif model_metadata_dict[cnn.CONV_2D3D_KEY]: generator_object = testing_io.myrorss_generator_2d3d( option_dict=training_option_dict, num_examples_total=LARGE_INTEGER) else: generator_object = testing_io.generator_2d_or_3d( option_dict=training_option_dict, num_examples_total=LARGE_INTEGER) include_soundings = (training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is not None) try: storm_object_dict = next(generator_object) except StopIteration: storm_object_dict = None print(SEPARATOR_STRING) if storm_object_dict is not None: observed_labels = storm_object_dict[testing_io.TARGET_ARRAY_KEY] list_of_predictor_matrices = storm_object_dict[ testing_io.INPUT_MATRICES_KEY] if include_soundings: sounding_matrix = list_of_predictor_matrices[-1] else: sounding_matrix = None if model_metadata_dict[cnn.CONV_2D3D_KEY]: if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]: class_probability_matrix = cnn.apply_2d_or_3d_cnn( model_object=model_object, radar_image_matrix=list_of_predictor_matrices[0], sounding_matrix=sounding_matrix, verbose=True) else: class_probability_matrix = cnn.apply_2d3d_cnn( model_object=model_object, reflectivity_matrix_dbz=list_of_predictor_matrices[0], azimuthal_shear_matrix_s01=list_of_predictor_matrices[1], sounding_matrix=sounding_matrix, verbose=True) else: class_probability_matrix = cnn.apply_2d_or_3d_cnn( model_object=model_object, radar_image_matrix=list_of_predictor_matrices[0], sounding_matrix=sounding_matrix, verbose=True) print(SEPARATOR_STRING) num_examples = class_probability_matrix.shape[0] for k in [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]: print( '{0:d}th percentile of {1:d} forecast probs = {2:.4f}'.format( k, num_examples, numpy.percentile(class_probability_matrix[:, 1], k))) print('\n') target_param_dict = target_val_utils.target_name_to_params( training_option_dict[trainval_io.TARGET_NAME_KEY]) event_type_string = target_param_dict[target_val_utils.EVENT_TYPE_KEY] if event_type_string == linkage.TORNADO_EVENT_STRING: genesis_only = False elif event_type_string == linkage.TORNADOGENESIS_EVENT_STRING: genesis_only = True else: genesis_only = None target_name = target_val_utils.target_params_to_name( min_lead_time_sec=target_param_dict[ target_val_utils.MIN_LEAD_TIME_KEY], max_lead_time_sec=target_param_dict[ target_val_utils.MAX_LEAD_TIME_KEY], min_link_distance_metres=target_param_dict[ target_val_utils.MIN_LINKAGE_DISTANCE_KEY], max_link_distance_metres=10000., genesis_only=genesis_only) output_file_name = prediction_io.find_file( top_prediction_dir_name=top_output_dir_name, first_init_time_unix_sec=first_time_unix_sec, last_init_time_unix_sec=last_time_unix_sec, gridded=False, raise_error_if_missing=False) print('Writing "{0:s}" predictions to: "{1:s}"...'.format( target_name, output_file_name)) if storm_object_dict is None: num_output_neurons = ( model_object.layers[-1].output.get_shape().as_list()[-1]) num_classes = max([num_output_neurons, 2]) class_probability_matrix = numpy.full((0, num_classes), numpy.nan) prediction_io.write_ungridded_predictions( netcdf_file_name=output_file_name, class_probability_matrix=class_probability_matrix, storm_ids=[], storm_times_unix_sec=numpy.array([], dtype=int), target_name=target_name, observed_labels=numpy.array([], dtype=int)) return prediction_io.write_ungridded_predictions( netcdf_file_name=output_file_name, class_probability_matrix=class_probability_matrix, storm_ids=storm_object_dict[testing_io.FULL_IDS_KEY], storm_times_unix_sec=storm_object_dict[testing_io.STORM_TIMES_KEY], target_name=target_name, observed_labels=observed_labels)
def _run(input_cnn_file_name, input_upconvnet_file_name, cnn_feature_layer_name, top_training_dir_name, first_training_time_string, last_training_time_string, top_validation_dir_name, first_validation_time_string, last_validation_time_string, num_examples_per_batch, num_epochs, num_training_batches_per_epoch, num_validation_batches_per_epoch, output_model_file_name): """Trains upconvnet. This is effectively the main method. :param input_cnn_file_name: See documentation at top of file. :param input_upconvnet_file_name: Same. :param cnn_feature_layer_name: Same. :param top_training_dir_name: Same. :param first_training_time_string: Same. :param last_training_time_string: Same. :param top_validation_dir_name: Same. :param first_validation_time_string: Same. :param last_validation_time_string: Same. :param num_examples_per_batch: Same. :param num_epochs: Same. :param num_training_batches_per_epoch: Same. :param num_validation_batches_per_epoch: Same. :param output_model_file_name: Same. """ # Find training and validation files. first_training_time_unix_sec = time_conversion.string_to_unix_sec( first_training_time_string, TIME_FORMAT) last_training_time_unix_sec = time_conversion.string_to_unix_sec( last_training_time_string, TIME_FORMAT) first_validation_time_unix_sec = time_conversion.string_to_unix_sec( first_validation_time_string, TIME_FORMAT) last_validation_time_unix_sec = time_conversion.string_to_unix_sec( last_validation_time_string, TIME_FORMAT) training_file_names = input_examples.find_many_example_files( top_directory_name=top_training_dir_name, shuffled=True, first_batch_number=FIRST_BATCH_NUMBER, last_batch_number=LAST_BATCH_NUMBER, raise_error_if_any_missing=False) validation_file_names = input_examples.find_many_example_files( top_directory_name=top_validation_dir_name, shuffled=True, first_batch_number=FIRST_BATCH_NUMBER, last_batch_number=LAST_BATCH_NUMBER, raise_error_if_any_missing=False) print 'Reading trained CNN from: "{0:s}"...'.format(input_cnn_file_name) cnn_model_object = cnn.read_model(input_cnn_file_name) cnn_model_object.summary() print SEPARATOR_STRING cnn_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(input_cnn_file_name)[0]) print 'Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name) cnn_metadata_dict = cnn.read_model_metadata(cnn_metafile_name) print 'Reading upconvnet architecture from: "{0:s}"...'.format( input_upconvnet_file_name) upconvnet_model_object = cnn.read_model(input_upconvnet_file_name) upconvnet_model_object = keras.models.clone_model(upconvnet_model_object) # TODO(thunderhoser): This is a HACK. upconvnet_model_object.compile(loss=keras.losses.mean_squared_error, optimizer=keras.optimizers.Adam()) print SEPARATOR_STRING upconvnet_model_object.summary() print SEPARATOR_STRING upconvnet_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(output_model_file_name)[0]) print 'Writing upconvnet metadata to: "{0:s}"...'.format( upconvnet_metafile_name) upconvnet.write_model_metadata( cnn_file_name=input_cnn_file_name, cnn_feature_layer_name=cnn_feature_layer_name, num_epochs=num_epochs, num_examples_per_batch=num_examples_per_batch, num_training_batches_per_epoch=num_training_batches_per_epoch, training_example_file_names=training_file_names, first_training_time_unix_sec=first_training_time_unix_sec, last_training_time_unix_sec=last_training_time_unix_sec, num_validation_batches_per_epoch=num_validation_batches_per_epoch, validation_example_file_names=validation_file_names, first_validation_time_unix_sec=first_validation_time_unix_sec, last_validation_time_unix_sec=last_validation_time_unix_sec, pickle_file_name=upconvnet_metafile_name) print SEPARATOR_STRING upconvnet.train_upconvnet( upconvnet_model_object=upconvnet_model_object, output_model_file_name=output_model_file_name, cnn_model_object=cnn_model_object, cnn_feature_layer_name=cnn_feature_layer_name, cnn_metadata_dict=cnn_metadata_dict, num_epochs=num_epochs, num_examples_per_batch=num_examples_per_batch, num_training_batches_per_epoch=num_training_batches_per_epoch, training_example_file_names=training_file_names, first_training_time_unix_sec=first_training_time_unix_sec, last_training_time_unix_sec=last_training_time_unix_sec, num_validation_batches_per_epoch=num_validation_batches_per_epoch, validation_example_file_names=validation_file_names, first_validation_time_unix_sec=first_validation_time_unix_sec, last_validation_time_unix_sec=last_validation_time_unix_sec)
def _run(model_file_name, target_class, target_layer_name, top_example_dir_name, storm_metafile_name, num_examples, randomize_weights, cascading_random, output_file_name): """Runs Grad-CAM (gradient-weighted class-activation maps). This is effectively the main method. :param model_file_name: See documentation at top of file. :param target_class: Same. :param target_layer_name: Same. :param top_example_dir_name: Same. :param storm_metafile_name: Same. :param num_examples: Same. :param randomize_weights: Same. :param cascading_random: Same. :param output_file_name: Same. """ file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) # Read model and metadata. print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = cnn.read_model(model_file_name) model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0]) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None output_dir_name, pathless_output_file_name = os.path.split( output_file_name) extensionless_output_file_name, output_file_extension = os.path.splitext( pathless_output_file_name) if randomize_weights: conv_dense_layer_names = _find_conv_and_dense_layers(model_object) conv_dense_layer_names.reverse() num_sets = len(conv_dense_layer_names) else: conv_dense_layer_names = [] num_sets = 1 print( 'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name)) full_storm_id_strings, storm_times_unix_sec = ( tracking_io.read_ids_and_times(storm_metafile_name)) print(SEPARATOR_STRING) if 0 < num_examples < len(full_storm_id_strings): full_storm_id_strings = full_storm_id_strings[:num_examples] storm_times_unix_sec = storm_times_unix_sec[:num_examples] example_dict = testing_io.read_predictors_specific_examples( top_example_dir_name=top_example_dir_name, desired_full_id_strings=full_storm_id_strings, desired_times_unix_sec=storm_times_unix_sec, option_dict=training_option_dict, layer_operation_dicts=model_metadata_dict[cnn.LAYER_OPERATIONS_KEY]) print(SEPARATOR_STRING) predictor_matrices = example_dict[testing_io.INPUT_MATRICES_KEY] sounding_pressure_matrix_pa = ( example_dict[testing_io.SOUNDING_PRESSURES_KEY]) print('Denormalizing model inputs...') denorm_predictor_matrices = trainval_io.separate_shear_and_reflectivity( list_of_input_matrices=copy.deepcopy(predictor_matrices), training_option_dict=training_option_dict) denorm_predictor_matrices = model_interpretation.denormalize_data( list_of_input_matrices=denorm_predictor_matrices, model_metadata_dict=model_metadata_dict) print(SEPARATOR_STRING) for k in range(num_sets): if randomize_weights: if cascading_random: _reset_weights_in_layer(model_object=model_object, layer_name=conv_dense_layer_names[k]) this_model_object = model_object this_output_file_name = ( '{0:s}/{1:s}_cascading-random_{2:s}{3:s}').format( output_dir_name, extensionless_output_file_name, conv_dense_layer_names[k].replace('_', '-'), output_file_extension) else: this_model_object = keras.models.Model.from_config( model_object.get_config()) this_model_object.set_weights(model_object.get_weights()) _reset_weights_in_layer(model_object=this_model_object, layer_name=conv_dense_layer_names[k]) this_output_file_name = '{0:s}/{1:s}_random_{2:s}{3:s}'.format( output_dir_name, extensionless_output_file_name, conv_dense_layer_names[k].replace('_', '-'), output_file_extension) else: this_model_object = model_object this_output_file_name = output_file_name # print(K.eval(this_model_object.get_layer(name='dense_53').weights[0])) these_cam_matrices, these_guided_cam_matrices = ( _run_gradcam_one_weight_set( model_object=this_model_object, target_class=target_class, target_layer_name=target_layer_name, predictor_matrices=predictor_matrices, training_option_dict=training_option_dict)) print('Writing results to file: "{0:s}"...'.format( this_output_file_name)) gradcam.write_standard_file( pickle_file_name=this_output_file_name, denorm_predictor_matrices=denorm_predictor_matrices, cam_matrices=these_cam_matrices, guided_cam_matrices=these_guided_cam_matrices, full_storm_id_strings=full_storm_id_strings, storm_times_unix_sec=storm_times_unix_sec, model_file_name=model_file_name, target_class=target_class, target_layer_name=target_layer_name, sounding_pressure_matrix_pa=sounding_pressure_matrix_pa) print(SEPARATOR_STRING)
def _run(cnn_file_name, upconvnet_file_name, top_example_dir_name, baseline_storm_metafile_name, trial_storm_metafile_name, num_baseline_examples, num_trial_examples, num_novel_examples, cnn_feature_layer_name, percent_svd_variance_to_keep, output_file_name): """Runs novelty detection. This is effectively the main method. :param cnn_file_name: See documentation at top of file. :param upconvnet_file_name: Same. :param top_example_dir_name: Same. :param baseline_storm_metafile_name: Same. :param trial_storm_metafile_name: Same. :param num_baseline_examples: Same. :param num_trial_examples: Same. :param num_novel_examples: Same. :param cnn_feature_layer_name: Same. :param percent_svd_variance_to_keep: Same. :param output_file_name: Same. :raises: ValueError: if dimensions of first CNN input matrix != dimensions of upconvnet output. """ print('Reading trained CNN from: "{0:s}"...'.format(cnn_file_name)) cnn_model_object = cnn.read_model(cnn_file_name) cnn_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(cnn_file_name)[0] ) print('Reading trained upconvnet from: "{0:s}"...'.format( upconvnet_file_name)) upconvnet_model_object = cnn.read_model(upconvnet_file_name) # ucn_output_dimensions = numpy.array( # upconvnet_model_object.output.get_shape().as_list()[1:], dtype=int # ) if isinstance(cnn_model_object.input, list): first_cnn_input_tensor = cnn_model_object.input[0] else: first_cnn_input_tensor = cnn_model_object.input cnn_input_dimensions = numpy.array( first_cnn_input_tensor.get_shape().as_list()[1:], dtype=int ) # if not numpy.array_equal(cnn_input_dimensions, ucn_output_dimensions): # error_string = ( # 'Dimensions of first CNN input matrix ({0:s}) should equal ' # 'dimensions of upconvnet output ({1:s}).' # ).format(str(cnn_input_dimensions), str(ucn_output_dimensions)) # # raise ValueError(error_string) print('Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name)) cnn_metadata_dict = cnn.read_model_metadata(cnn_metafile_name) print('Reading metadata for baseline examples from: "{0:s}"...'.format( baseline_storm_metafile_name)) baseline_full_id_strings, baseline_times_unix_sec = ( tracking_io.read_ids_and_times(baseline_storm_metafile_name) ) print('Reading metadata for trial examples from: "{0:s}"...'.format( trial_storm_metafile_name)) trial_full_id_strings, trial_times_unix_sec = ( tracking_io.read_ids_and_times(trial_storm_metafile_name) ) if 0 < num_baseline_examples < len(baseline_full_id_strings): baseline_full_id_strings = baseline_full_id_strings[ :num_baseline_examples] baseline_times_unix_sec = baseline_times_unix_sec[ :num_baseline_examples] if 0 < num_trial_examples < len(trial_full_id_strings): trial_full_id_strings = trial_full_id_strings[:num_trial_examples] trial_times_unix_sec = trial_times_unix_sec[:num_trial_examples] num_trial_examples = len(trial_full_id_strings) if num_novel_examples <= 0: num_novel_examples = num_trial_examples + 0 num_novel_examples = min([num_novel_examples, num_trial_examples]) print('Number of novel examples to find: {0:d}'.format(num_novel_examples)) bad_baseline_indices = tracking_utils.find_storm_objects( all_id_strings=baseline_full_id_strings, all_times_unix_sec=baseline_times_unix_sec, id_strings_to_keep=trial_full_id_strings, times_to_keep_unix_sec=trial_times_unix_sec, allow_missing=True) print('Removing {0:d} trial examples from baseline set...'.format( len(bad_baseline_indices) )) baseline_times_unix_sec = numpy.delete( baseline_times_unix_sec, bad_baseline_indices ) baseline_full_id_strings = numpy.delete( numpy.array(baseline_full_id_strings), bad_baseline_indices ) baseline_full_id_strings = baseline_full_id_strings.tolist() # num_baseline_examples = len(baseline_full_id_strings) print(SEPARATOR_STRING) list_of_baseline_input_matrices, _ = testing_io.read_specific_examples( top_example_dir_name=top_example_dir_name, desired_full_id_strings=baseline_full_id_strings, desired_times_unix_sec=baseline_times_unix_sec, option_dict=cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY], list_of_layer_operation_dicts=cnn_metadata_dict[ cnn.LAYER_OPERATIONS_KEY] ) print(SEPARATOR_STRING) list_of_trial_input_matrices, _ = testing_io.read_specific_examples( top_example_dir_name=top_example_dir_name, desired_full_id_strings=trial_full_id_strings, desired_times_unix_sec=trial_times_unix_sec, option_dict=cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY], list_of_layer_operation_dicts=cnn_metadata_dict[ cnn.LAYER_OPERATIONS_KEY] ) print(SEPARATOR_STRING) novelty_dict = novelty_detection.do_novelty_detection( list_of_baseline_input_matrices=list_of_baseline_input_matrices, list_of_trial_input_matrices=list_of_trial_input_matrices, cnn_model_object=cnn_model_object, cnn_feature_layer_name=cnn_feature_layer_name, upconvnet_model_object=upconvnet_model_object, num_novel_examples=num_novel_examples, multipass=False, percent_svd_variance_to_keep=percent_svd_variance_to_keep) print(SEPARATOR_STRING) print('Adding metadata to novelty-detection results...') novelty_dict = novelty_detection.add_metadata( novelty_dict=novelty_dict, baseline_full_id_strings=baseline_full_id_strings, baseline_storm_times_unix_sec=baseline_times_unix_sec, trial_full_id_strings=trial_full_id_strings, trial_storm_times_unix_sec=trial_times_unix_sec, cnn_file_name=cnn_file_name, upconvnet_file_name=upconvnet_file_name) print('Denormalizing inputs and outputs of novelty detection...') novelty_dict[novelty_detection.BASELINE_INPUTS_KEY] = ( model_interpretation.denormalize_data( list_of_input_matrices=novelty_dict[ novelty_detection.BASELINE_INPUTS_KEY ], model_metadata_dict=cnn_metadata_dict) ) novelty_dict[novelty_detection.TRIAL_INPUTS_KEY] = ( model_interpretation.denormalize_data( list_of_input_matrices=novelty_dict[ novelty_detection.TRIAL_INPUTS_KEY ], model_metadata_dict=cnn_metadata_dict) ) cnn_metadata_dict[ cnn.TRAINING_OPTION_DICT_KEY][trainval_io.SOUNDING_FIELDS_KEY] = None novelty_dict[novelty_detection.NOVEL_IMAGES_UPCONV_KEY] = ( model_interpretation.denormalize_data( list_of_input_matrices=[ novelty_dict[novelty_detection.NOVEL_IMAGES_UPCONV_KEY] ], model_metadata_dict=cnn_metadata_dict) )[0] novelty_dict[novelty_detection.NOVEL_IMAGES_UPCONV_SVD_KEY] = ( model_interpretation.denormalize_data( list_of_input_matrices=[ novelty_dict[novelty_detection.NOVEL_IMAGES_UPCONV_SVD_KEY] ], model_metadata_dict=cnn_metadata_dict) )[0] print('Writing results to: "{0:s}"...'.format(output_file_name)) novelty_detection.write_standard_file(novelty_dict=novelty_dict, pickle_file_name=output_file_name)
def _run(model_file_name, top_example_dir_name, first_spc_date_string, last_spc_date_string, num_examples, do_backwards_test, separate_radar_heights, downsampling_keys, downsampling_values, num_bootstrap_reps, output_file_name): """Runs permutation test for predictor importance. This is effectively the main method. :param model_file_name: See documentation at top of file. :param top_example_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param num_examples: Same. :param do_backwards_test: Same. :param separate_radar_heights: Same. :param downsampling_keys: Same. :param downsampling_values: Same. :param num_bootstrap_reps: Same. :param output_file_name: Same. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = cnn.read_model(model_file_name) metafile_name = cnn.find_metafile(model_file_name=model_file_name) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) cnn_metadata_dict = cnn.read_model_metadata(metafile_name) if len(downsampling_keys) > 1: downsampling_dict = dict( list(zip(downsampling_keys, downsampling_values))) else: downsampling_dict = None training_option_dict = cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] training_option_dict[ trainval_io.SAMPLING_FRACTIONS_KEY] = downsampling_dict example_file_names = input_examples.find_many_example_files( top_directory_name=top_example_dir_name, shuffled=False, first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string, raise_error_if_any_missing=False) training_option_dict[trainval_io.EXAMPLE_FILES_KEY] = example_file_names training_option_dict[trainval_io.FIRST_STORM_TIME_KEY] = ( time_conversion.get_start_of_spc_date(first_spc_date_string)) training_option_dict[trainval_io.LAST_STORM_TIME_KEY] = ( time_conversion.get_end_of_spc_date(last_spc_date_string)) training_option_dict[trainval_io.NUM_EXAMPLES_PER_BATCH_KEY] = ( NUM_EXAMPLES_PER_BATCH) if cnn_metadata_dict[cnn.LAYER_OPERATIONS_KEY] is not None: generator_object = testing_io.gridrad_generator_2d_reduced( option_dict=training_option_dict, desired_num_examples=num_examples, list_of_operation_dicts=cnn_metadata_dict[ cnn.LAYER_OPERATIONS_KEY]) elif cnn_metadata_dict[cnn.CONV_2D3D_KEY]: generator_object = testing_io.myrorss_generator_2d3d( option_dict=training_option_dict, desired_num_examples=num_examples) else: generator_object = testing_io.generator_2d_or_3d( option_dict=training_option_dict, desired_num_examples=num_examples) full_storm_id_strings = [] storm_times_unix_sec = numpy.array([], dtype=int) target_values = numpy.array([], dtype=int) predictor_matrices = None print(SEPARATOR_STRING) for _ in range(len(example_file_names)): try: this_storm_object_dict = next(generator_object) print(SEPARATOR_STRING) except StopIteration: break full_storm_id_strings += this_storm_object_dict[ testing_io.FULL_IDS_KEY] storm_times_unix_sec = numpy.concatenate( (storm_times_unix_sec, this_storm_object_dict[testing_io.STORM_TIMES_KEY])) these_target_values = this_storm_object_dict[ testing_io.TARGET_ARRAY_KEY] if len(these_target_values.shape) > 1: these_target_values = numpy.argmax(these_target_values, axis=1) target_values = numpy.concatenate((target_values, these_target_values)) these_predictor_matrices = this_storm_object_dict[ testing_io.INPUT_MATRICES_KEY] if predictor_matrices is None: predictor_matrices = copy.deepcopy(these_predictor_matrices) else: for k in range(len(predictor_matrices)): predictor_matrices[k] = numpy.concatenate( (predictor_matrices[k], these_predictor_matrices[k])) print(SEPARATOR_STRING) correlation_matrix, predictor_names = correlation.get_pearson_correlations( predictor_matrices=predictor_matrices, cnn_metadata_dict=cnn_metadata_dict, separate_radar_heights=separate_radar_heights) print(SEPARATOR_STRING) num_predictors = len(predictor_names) for i in range(num_predictors): for j in range(i, num_predictors): print(('Pearson correlation between "{0:s}" and "{1:s}" = {2:.3f}' ).format(predictor_names[i], predictor_names[j], correlation_matrix[i, j])) print(SEPARATOR_STRING) if do_backwards_test: result_dict = permutation.run_backwards_test( model_object=model_object, predictor_matrices=predictor_matrices, target_values=target_values, cnn_metadata_dict=cnn_metadata_dict, cost_function=permutation_utils.negative_auc_function, separate_radar_heights=separate_radar_heights, num_bootstrap_reps=num_bootstrap_reps) else: result_dict = permutation.run_forward_test( model_object=model_object, predictor_matrices=predictor_matrices, target_values=target_values, cnn_metadata_dict=cnn_metadata_dict, cost_function=permutation_utils.negative_auc_function, separate_radar_heights=separate_radar_heights, num_bootstrap_reps=num_bootstrap_reps) print(SEPARATOR_STRING) result_dict[permutation_utils.MODEL_FILE_KEY] = model_file_name result_dict[permutation_utils.TARGET_VALUES_KEY] = target_values result_dict[permutation_utils.FULL_IDS_KEY] = full_storm_id_strings result_dict[permutation_utils.STORM_TIMES_KEY] = storm_times_unix_sec print('Writing results to: "{0:s}"...'.format(output_file_name)) permutation_utils.write_results(result_dict=result_dict, pickle_file_name=output_file_name)
def _run(input_human_file_name, input_machine_file_name, guided_gradcam_flag, abs_percentile_threshold, output_dir_name): """Compares human-generated vs. machine-generated interpretation map. This is effectively the main method. :param input_human_file_name: See documentation at top of file. :param input_machine_file_name: Same. :param guided_gradcam_flag: Same. :param abs_percentile_threshold: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) if abs_percentile_threshold < 0: abs_percentile_threshold = None if abs_percentile_threshold is not None: error_checking.assert_is_leq(abs_percentile_threshold, 100.) print('Reading data from: "{0:s}"...'.format(input_human_file_name)) human_polygon_dict = human_polygons.read_polygons(input_human_file_name) human_positive_mask_matrix_4d = human_polygon_dict[ human_polygons.POSITIVE_MASK_MATRIX_KEY] human_negative_mask_matrix_4d = human_polygon_dict[ human_polygons.NEGATIVE_MASK_MATRIX_KEY] full_storm_id_string = human_polygon_dict[human_polygons.STORM_ID_KEY] storm_time_unix_sec = human_polygon_dict[human_polygons.STORM_TIME_KEY] pmm_flag = full_storm_id_string is None and storm_time_unix_sec is None print('Reading data from: "{0:s}"...'.format(input_machine_file_name)) # TODO(thunderhoser): This is a HACK. machine_channel_indices = numpy.array([2, 8], dtype=int) if pmm_flag: try: saliency_dict = saliency_maps.read_pmm_file(input_machine_file_name) saliency_flag = True model_file_name = saliency_dict[saliency_maps.MODEL_FILE_KEY] predictor_matrix = saliency_dict.pop( saliency_maps.MEAN_INPUT_MATRICES_KEY )[0][..., machine_channel_indices] machine_interpretation_matrix_3d = saliency_dict.pop( saliency_maps.MEAN_SALIENCY_MATRICES_KEY )[0][..., machine_channel_indices] except ValueError: gradcam_dict = gradcam.read_pmm_file(input_machine_file_name) saliency_flag = False model_file_name = gradcam_dict[gradcam.MODEL_FILE_KEY] predictor_matrix = gradcam_dict.pop( gradcam.MEAN_INPUT_MATRICES_KEY )[0][..., machine_channel_indices] if guided_gradcam_flag: machine_interpretation_matrix_3d = gradcam_dict.pop( gradcam.MEAN_GUIDED_GRADCAM_KEY )[..., machine_channel_indices] else: machine_interpretation_matrix_3d = gradcam_dict.pop( gradcam.MEAN_CLASS_ACTIVATIONS_KEY) else: try: saliency_dict = saliency_maps.read_standard_file( input_machine_file_name) saliency_flag = True all_full_id_strings = saliency_dict[saliency_maps.FULL_IDS_KEY] all_times_unix_sec = saliency_dict[saliency_maps.STORM_TIMES_KEY] model_file_name = saliency_dict[saliency_maps.MODEL_FILE_KEY] predictor_matrix = saliency_dict.pop( saliency_maps.INPUT_MATRICES_KEY )[0][..., machine_channel_indices] machine_interpretation_matrix_3d = saliency_dict.pop( saliency_maps.SALIENCY_MATRICES_KEY )[0][..., machine_channel_indices] except ValueError: gradcam_dict = gradcam.read_standard_file(input_machine_file_name) saliency_flag = False all_full_id_strings = gradcam_dict[gradcam.FULL_IDS_KEY] all_times_unix_sec = gradcam_dict[gradcam.STORM_TIMES_KEY] model_file_name = gradcam_dict[gradcam.MODEL_FILE_KEY] predictor_matrix = gradcam_dict.pop( gradcam.INPUT_MATRICES_KEY )[0][..., machine_channel_indices] if guided_gradcam_flag: machine_interpretation_matrix_3d = gradcam_dict.pop( gradcam.GUIDED_GRADCAM_KEY )[..., machine_channel_indices] else: machine_interpretation_matrix_3d = gradcam_dict.pop( gradcam.CLASS_ACTIVATIONS_KEY) storm_object_index = tracking_utils.find_storm_objects( all_id_strings=all_full_id_strings, all_times_unix_sec=all_times_unix_sec, id_strings_to_keep=[full_storm_id_string], times_to_keep_unix_sec=numpy.array( [storm_time_unix_sec], dtype=int ), allow_missing=False )[0] predictor_matrix = predictor_matrix[storm_object_index, ...] machine_interpretation_matrix_3d = machine_interpretation_matrix_3d[ storm_object_index, ...] if not saliency_flag and not guided_gradcam_flag: machine_interpretation_matrix_3d = numpy.expand_dims( machine_interpretation_matrix_3d, axis=-1) machine_interpretation_matrix_3d = numpy.repeat( a=machine_interpretation_matrix_3d, repeats=predictor_matrix.shape[-1], axis=-1) if not (saliency_flag or guided_gradcam_flag): human_negative_mask_matrix_4d = None model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0] ) print('Reading metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) model_metadata_dict[cnn.LAYER_OPERATIONS_KEY] = [ model_metadata_dict[cnn.LAYER_OPERATIONS_KEY][k] for k in machine_channel_indices ] human_positive_mask_matrix_3d, human_negative_mask_matrix_3d = ( _reshape_human_maps( model_metadata_dict=model_metadata_dict, positive_mask_matrix_4d=human_positive_mask_matrix_4d, negative_mask_matrix_4d=human_negative_mask_matrix_4d) ) num_channels = human_positive_mask_matrix_3d.shape[-1] machine_positive_mask_matrix_3d = numpy.full( human_positive_mask_matrix_3d.shape, False, dtype=bool) positive_iou_by_channel = numpy.full(num_channels, numpy.nan) if human_negative_mask_matrix_3d is None: machine_negative_mask_matrix_3d = None negative_iou_by_channel = None else: machine_negative_mask_matrix_3d = numpy.full( human_negative_mask_matrix_3d.shape, False, dtype=bool) negative_iou_by_channel = numpy.full(num_channels, numpy.nan) for k in range(num_channels): this_negative_matrix = ( None if human_negative_mask_matrix_3d is None else human_negative_mask_matrix_3d[..., k] ) this_comparison_dict = _do_comparison_one_channel( machine_interpretation_matrix_2d=machine_interpretation_matrix_3d[ ..., k], abs_percentile_threshold=abs_percentile_threshold, human_positive_mask_matrix_2d=human_positive_mask_matrix_3d[..., k], human_negative_mask_matrix_2d=this_negative_matrix) machine_positive_mask_matrix_3d[..., k] = this_comparison_dict[ MACHINE_POSITIVE_MASK_KEY] positive_iou_by_channel[k] = this_comparison_dict[POSITIVE_IOU_KEY] if human_negative_mask_matrix_3d is None: continue machine_negative_mask_matrix_3d[..., k] = this_comparison_dict[ MACHINE_NEGATIVE_MASK_KEY] negative_iou_by_channel[k] = this_comparison_dict[NEGATIVE_IOU_KEY] this_file_name = '{0:s}/positive_comparison.jpg'.format(output_dir_name) _plot_comparison( predictor_matrix=predictor_matrix, model_metadata_dict=model_metadata_dict, machine_mask_matrix_3d=machine_positive_mask_matrix_3d, human_mask_matrix_3d=human_positive_mask_matrix_3d, iou_by_channel=positive_iou_by_channel, positive_flag=True, output_file_name=this_file_name) if human_negative_mask_matrix_3d is None: return this_file_name = '{0:s}/negative_comparison.jpg'.format(output_dir_name) _plot_comparison( predictor_matrix=predictor_matrix, model_metadata_dict=model_metadata_dict, machine_mask_matrix_3d=machine_negative_mask_matrix_3d, human_mask_matrix_3d=human_negative_mask_matrix_3d, iou_by_channel=negative_iou_by_channel, positive_flag=False, output_file_name=this_file_name)
def _run(input_cnn_file_name, input_upconvnet_file_name, cnn_feature_layer_name, top_training_dir_name, first_training_time_string, last_training_time_string, top_validation_dir_name, first_validation_time_string, last_validation_time_string, num_examples_per_batch, num_epochs, num_training_batches_per_epoch, num_validation_batches_per_epoch, output_dir_name): """Trains upconvnet. This is effectively the main method. :param input_cnn_file_name: See documentation at top of file. :param input_upconvnet_file_name: Same. :param cnn_feature_layer_name: Same. :param top_training_dir_name: Same. :param first_training_time_string: Same. :param last_training_time_string: Same. :param top_validation_dir_name: Same. :param first_validation_time_string: Same. :param last_validation_time_string: Same. :param num_examples_per_batch: Same. :param num_epochs: Same. :param num_training_batches_per_epoch: Same. :param num_validation_batches_per_epoch: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) # argument_file_name = '{0:s}/input_args.p'.format(output_dir_name) # print('Writing input args to: "{0:s}"...'.format(argument_file_name)) # # argument_file_handle = open(argument_file_name, 'wb') # pickle.dump(INPUT_ARG_OBJECT.__dict__, argument_file_handle) # argument_file_handle.close() # # return # Process input args. first_training_time_unix_sec = time_conversion.string_to_unix_sec( first_training_time_string, TIME_FORMAT) last_training_time_unix_sec = time_conversion.string_to_unix_sec( last_training_time_string, TIME_FORMAT) first_validation_time_unix_sec = time_conversion.string_to_unix_sec( first_validation_time_string, TIME_FORMAT) last_validation_time_unix_sec = time_conversion.string_to_unix_sec( last_validation_time_string, TIME_FORMAT) # Find training and validation files. training_file_names = input_examples.find_many_example_files( top_directory_name=top_training_dir_name, shuffled=True, first_batch_number=FIRST_BATCH_NUMBER, last_batch_number=LAST_BATCH_NUMBER, raise_error_if_any_missing=False) validation_file_names = input_examples.find_many_example_files( top_directory_name=top_validation_dir_name, shuffled=True, first_batch_number=FIRST_BATCH_NUMBER, last_batch_number=LAST_BATCH_NUMBER, raise_error_if_any_missing=False) # Read trained CNN. print('Reading trained CNN from: "{0:s}"...'.format(input_cnn_file_name)) cnn_model_object = cnn.read_model(input_cnn_file_name) cnn_model_object.summary() print(SEPARATOR_STRING) cnn_metafile_name = cnn.find_metafile(model_file_name=input_cnn_file_name, raise_error_if_missing=True) print('Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name)) cnn_metadata_dict = cnn.read_model_metadata(cnn_metafile_name) # Read architecture. print('Reading upconvnet architecture from: "{0:s}"...'.format( input_upconvnet_file_name)) upconvnet_model_object = cnn.read_model(input_upconvnet_file_name) # upconvnet_model_object = keras.models.clone_model(upconvnet_model_object) # TODO(thunderhoser): This is a HACK. upconvnet_model_object.compile(loss=keras.losses.mean_squared_error, optimizer=keras.optimizers.Adam()) upconvnet_model_object.summary() print(SEPARATOR_STRING) upconvnet_metafile_name = cnn.find_metafile( model_file_name='{0:s}/foo.h5'.format(output_dir_name), raise_error_if_missing=False) print('Writing upconvnet metadata to: "{0:s}"...'.format( upconvnet_metafile_name)) upconvnet.write_model_metadata( cnn_file_name=input_cnn_file_name, cnn_feature_layer_name=cnn_feature_layer_name, num_epochs=num_epochs, num_examples_per_batch=num_examples_per_batch, num_training_batches_per_epoch=num_training_batches_per_epoch, training_example_file_names=training_file_names, first_training_time_unix_sec=first_training_time_unix_sec, last_training_time_unix_sec=last_training_time_unix_sec, num_validation_batches_per_epoch=num_validation_batches_per_epoch, validation_example_file_names=validation_file_names, first_validation_time_unix_sec=first_validation_time_unix_sec, last_validation_time_unix_sec=last_validation_time_unix_sec, pickle_file_name=upconvnet_metafile_name) print(SEPARATOR_STRING) upconvnet.train_upconvnet( upconvnet_model_object=upconvnet_model_object, output_dir_name=output_dir_name, cnn_model_object=cnn_model_object, cnn_metadata_dict=cnn_metadata_dict, cnn_feature_layer_name=cnn_feature_layer_name, num_epochs=num_epochs, num_examples_per_batch=num_examples_per_batch, num_training_batches_per_epoch=num_training_batches_per_epoch, training_example_file_names=training_file_names, first_training_time_unix_sec=first_training_time_unix_sec, last_training_time_unix_sec=last_training_time_unix_sec, num_validation_batches_per_epoch=num_validation_batches_per_epoch, validation_example_file_names=validation_file_names, first_validation_time_unix_sec=first_validation_time_unix_sec, last_validation_time_unix_sec=last_validation_time_unix_sec)
def _run(input_file_name, colour_map_name, max_colour_value, half_num_contours, smoothing_radius_grid_cells, plot_soundings, allow_whitespace, plot_panel_names, add_titles, label_colour_bars, colour_bar_length, output_dir_name): """Plots saliency maps. This is effectively the main method. :param input_file_name: See documentation at top of file. :param colour_map_name: Same. :param max_colour_value: Same. :param half_num_contours: Same. :param smoothing_radius_grid_cells: Same. :param plot_soundings: Same. :param allow_whitespace: Same. :param plot_panel_names: Same. :param add_titles: Same. :param label_colour_bars: Same. :param colour_bar_length: Same. :param output_dir_name: Same. """ if max_colour_value <= 0: max_colour_value = None if smoothing_radius_grid_cells <= 0: smoothing_radius_grid_cells = None file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) colour_map_object = pyplot.cm.get_cmap(colour_map_name) error_checking.assert_is_geq(half_num_contours, 5) print('Reading data from: "{0:s}"...'.format(input_file_name)) saliency_dict, pmm_flag = saliency_maps.read_file(input_file_name) if pmm_flag: predictor_matrices = saliency_dict.pop( saliency_maps.MEAN_PREDICTOR_MATRICES_KEY) saliency_matrices = saliency_dict.pop( saliency_maps.MEAN_SALIENCY_MATRICES_KEY) full_storm_id_strings = [None] storm_times_unix_sec = [None] mean_sounding_pressures_pa = saliency_dict[ saliency_maps.MEAN_SOUNDING_PRESSURES_KEY] sounding_pressure_matrix_pa = numpy.reshape( mean_sounding_pressures_pa, (1, len(mean_sounding_pressures_pa)) ) for i in range(len(predictor_matrices)): predictor_matrices[i] = numpy.expand_dims( predictor_matrices[i], axis=0 ) saliency_matrices[i] = numpy.expand_dims( saliency_matrices[i], axis=0 ) else: predictor_matrices = saliency_dict.pop( saliency_maps.PREDICTOR_MATRICES_KEY) saliency_matrices = saliency_dict.pop( saliency_maps.SALIENCY_MATRICES_KEY) full_storm_id_strings = saliency_dict[saliency_maps.FULL_STORM_IDS_KEY] storm_times_unix_sec = saliency_dict[saliency_maps.STORM_TIMES_KEY] sounding_pressure_matrix_pa = saliency_dict[ saliency_maps.SOUNDING_PRESSURES_KEY] if smoothing_radius_grid_cells is not None: saliency_matrices = _smooth_maps( saliency_matrices=saliency_matrices, smoothing_radius_grid_cells=smoothing_radius_grid_cells) model_file_name = saliency_dict[saliency_maps.MODEL_FILE_KEY] model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0] ) print('Reading metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) print(SEPARATOR_STRING) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] num_radar_matrices = len(predictor_matrices) if training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is None: plot_soundings = False else: num_radar_matrices -= 1 num_examples = predictor_matrices[0].shape[0] for i in range(num_examples): this_handle_dict = plot_examples.plot_one_example( list_of_predictor_matrices=predictor_matrices, model_metadata_dict=model_metadata_dict, pmm_flag=pmm_flag, example_index=i, plot_sounding=plot_soundings, sounding_pressures_pascals=sounding_pressure_matrix_pa[i, ...], allow_whitespace=allow_whitespace, plot_panel_names=plot_panel_names, add_titles=add_titles, label_colour_bars=label_colour_bars, colour_bar_length=colour_bar_length) if plot_soundings: _plot_sounding_saliency( saliency_matrix=saliency_matrices[-1][i, ...], colour_map_object=colour_map_object, max_colour_value=max_colour_value, sounding_figure_object=this_handle_dict[ plot_examples.SOUNDING_FIGURE_KEY], sounding_axes_object=this_handle_dict[ plot_examples.SOUNDING_AXES_KEY], sounding_pressures_pascals=sounding_pressure_matrix_pa[i, ...], saliency_dict=saliency_dict, model_metadata_dict=model_metadata_dict, add_title=add_titles, output_dir_name=output_dir_name, pmm_flag=pmm_flag, example_index=i) these_figure_objects = this_handle_dict[plot_examples.RADAR_FIGURES_KEY] these_axes_object_matrices = this_handle_dict[ plot_examples.RADAR_AXES_KEY] for j in range(num_radar_matrices): this_num_spatial_dim = len(predictor_matrices[j].shape) - 2 if this_num_spatial_dim == 3: _plot_3d_radar_saliency( saliency_matrix=saliency_matrices[j][i, ...], colour_map_object=colour_map_object, max_colour_value=max_colour_value, half_num_contours=half_num_contours, label_colour_bars=label_colour_bars, colour_bar_length=colour_bar_length, figure_objects=these_figure_objects, axes_object_matrices=these_axes_object_matrices, model_metadata_dict=model_metadata_dict, output_dir_name=output_dir_name, significance_matrix=None, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i] ) else: _plot_2d_radar_saliency( saliency_matrix=saliency_matrices[j][i, ...], colour_map_object=colour_map_object, max_colour_value=max_colour_value, half_num_contours=half_num_contours, label_colour_bars=label_colour_bars, colour_bar_length=colour_bar_length, figure_objects=these_figure_objects, axes_object_matrices=these_axes_object_matrices, model_metadata_dict=model_metadata_dict, output_dir_name=output_dir_name, significance_matrix=None, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i] )
def _run(input_file_name, diff_colour_map_name, max_colour_percentile_for_diff, top_output_dir_name): """Plots results of novelty detection. This is effectively the main method. :param input_file_name: See documentation at top of file. :param diff_colour_map_name: Same. :param max_colour_percentile_for_diff: Same. :param top_output_dir_name: Same. """ actual_dir_name = '{0:s}/actual'.format(top_output_dir_name) upconv_dir_name = '{0:s}/upconvnet_reconstruction'.format( top_output_dir_name) upconv_svd_dir_name = '{0:s}/upconvnet_svd_reconstruction'.format( top_output_dir_name) difference_dir_name = '{0:s}/difference'.format(top_output_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=actual_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=upconv_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=upconv_svd_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=difference_dir_name) pmm_flag = False error_checking.assert_is_geq(max_colour_percentile_for_diff, 0.) error_checking.assert_is_leq(max_colour_percentile_for_diff, 100.) diff_colour_map_object = pyplot.cm.get_cmap(diff_colour_map_name) print('Reading data from: "{0:s}"...'.format(input_file_name)) try: novelty_dict = novelty_detection.read_standard_file(input_file_name) novel_indices = novelty_dict[novelty_detection.NOVEL_INDICES_KEY] novel_radar_matrix = novelty_dict.pop( novelty_detection.TRIAL_INPUTS_KEY)[0][novel_indices, ...] novel_radar_matrix_upconv = novelty_dict.pop( novelty_detection.NOVEL_IMAGES_UPCONV_KEY) novel_radar_matrix_upconv_svd = novelty_dict.pop( novelty_detection.NOVEL_IMAGES_UPCONV_SVD_KEY) novelty_metadata_dict = novelty_dict full_id_strings = novelty_metadata_dict[ novelty_detection.TRIAL_IDS_KEY] storm_time_strings = [ time_conversion.unix_sec_to_string(t, TIME_FORMAT) for t in novelty_metadata_dict[novelty_detection.TRIAL_STORM_TIMES_KEY] ] except ValueError: pmm_flag = True novelty_dict = novelty_detection.read_pmm_file(input_file_name) novel_radar_matrix = numpy.expand_dims(novelty_dict.pop( novelty_detection.MEAN_NOVEL_IMAGE_KEY), axis=0) novel_radar_matrix_upconv = numpy.expand_dims(novelty_dict.pop( novelty_detection.MEAN_NOVEL_IMAGE_UPCONV_KEY), axis=0) novel_radar_matrix_upconv_svd = numpy.expand_dims(novelty_dict.pop( novelty_detection.MEAN_NOVEL_IMAGE_UPCONV_SVD_KEY), axis=0) orig_novelty_file_name = novelty_dict[ novelty_detection.STANDARD_FILE_NAME_KEY] print( 'Reading metadata from: "{0:s}"...'.format(orig_novelty_file_name)) novelty_metadata_dict = novelty_detection.read_standard_file( orig_novelty_file_name) novelty_metadata_dict.pop(novelty_detection.TRIAL_INPUTS_KEY) novelty_metadata_dict.pop(novelty_detection.NOVEL_IMAGES_UPCONV_KEY) novelty_metadata_dict.pop( novelty_detection.NOVEL_IMAGES_UPCONV_SVD_KEY) full_id_strings = None storm_time_strings = None novelty_metadata_dict.pop(novelty_detection.BASELINE_INPUTS_KEY) cnn_file_name = novelty_metadata_dict[novelty_detection.CNN_FILE_KEY] cnn_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(cnn_file_name)[0]) print('Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name)) model_metadata_dict = cnn.read_model_metadata(cnn_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] print(SEPARATOR_STRING) num_radar_dimensions = len(novel_radar_matrix.shape) - 2 if num_radar_dimensions == 3: _plot_3d_radar(training_option_dict=training_option_dict, output_dir_name=actual_dir_name, pmm_flag=pmm_flag, full_id_strings=full_id_strings, storm_time_strings=storm_time_strings, novel_radar_matrix=novel_radar_matrix) print(SEPARATOR_STRING) _plot_3d_radar(training_option_dict=training_option_dict, output_dir_name=upconv_dir_name, pmm_flag=pmm_flag, full_id_strings=full_id_strings, storm_time_strings=storm_time_strings, novel_radar_matrix_upconv=novel_radar_matrix_upconv) print(SEPARATOR_STRING) _plot_3d_radar( training_option_dict=training_option_dict, output_dir_name=upconv_svd_dir_name, pmm_flag=pmm_flag, full_id_strings=full_id_strings, storm_time_strings=storm_time_strings, novel_radar_matrix_upconv_svd=novel_radar_matrix_upconv_svd) print(SEPARATOR_STRING) _plot_3d_radar( training_option_dict=training_option_dict, diff_colour_map_object=diff_colour_map_object, max_colour_percentile_for_diff=max_colour_percentile_for_diff, output_dir_name=difference_dir_name, pmm_flag=pmm_flag, full_id_strings=full_id_strings, storm_time_strings=storm_time_strings, novel_radar_matrix_upconv=novel_radar_matrix_upconv, novel_radar_matrix_upconv_svd=novel_radar_matrix_upconv_svd) return _plot_2d_radar(model_metadata_dict=model_metadata_dict, output_dir_name=actual_dir_name, pmm_flag=pmm_flag, full_id_strings=full_id_strings, storm_time_strings=storm_time_strings, novel_radar_matrix=novel_radar_matrix) print(SEPARATOR_STRING) _plot_2d_radar(model_metadata_dict=model_metadata_dict, output_dir_name=upconv_dir_name, pmm_flag=pmm_flag, full_id_strings=full_id_strings, storm_time_strings=storm_time_strings, novel_radar_matrix_upconv=novel_radar_matrix_upconv) print(SEPARATOR_STRING) _plot_2d_radar(model_metadata_dict=model_metadata_dict, output_dir_name=upconv_svd_dir_name, pmm_flag=pmm_flag, full_id_strings=full_id_strings, storm_time_strings=storm_time_strings, novel_radar_matrix_upconv_svd=novel_radar_matrix_upconv_svd) print(SEPARATOR_STRING) _plot_2d_radar( model_metadata_dict=model_metadata_dict, diff_colour_map_object=diff_colour_map_object, max_colour_percentile_for_diff=max_colour_percentile_for_diff, output_dir_name=difference_dir_name, pmm_flag=pmm_flag, full_id_strings=full_id_strings, storm_time_strings=storm_time_strings, novel_radar_matrix_upconv=novel_radar_matrix_upconv, novel_radar_matrix_upconv_svd=novel_radar_matrix_upconv_svd)
def _run(model_file_name, top_example_dir_name, first_spc_date_string, last_spc_date_string, num_examples, num_bootstrap_reps, confidence_level, class_fraction_keys, class_fraction_values, output_dir_name): """Evaluates CNN (convolutional neural net) predictions. This is effectively the main method. :param model_file_name: See documentation at top of file. :param top_example_dir_name: Same. :param first_spc_date_string: Same. :param last_spc_date_string: Same. :param num_examples: Same. :param num_bootstrap_reps: Same. :param confidence_level: Same. :param class_fraction_keys: Same. :param class_fraction_values: Same. :param output_dir_name: Same. :raises: ValueError: if the model does multi-class classification. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = cnn.read_model(model_file_name) num_output_neurons = ( model_object.layers[-1].output.get_shape().as_list()[-1]) if num_output_neurons > 2: error_string = ( 'The model has {0:d} output neurons, which suggests {0:d}-class ' 'classification. This script handles only binary classification.' ).format(num_output_neurons) raise ValueError(error_string) soundings_only = False if isinstance(model_object.input, list): list_of_input_tensors = model_object.input else: list_of_input_tensors = [model_object.input] if len(list_of_input_tensors) == 1: these_spatial_dim = numpy.array( list_of_input_tensors[0].get_shape().as_list()[1:-1], dtype=int) soundings_only = len(these_spatial_dim) == 1 model_directory_name, _ = os.path.split(model_file_name) model_metafile_name = '{0:s}/model_metadata.p'.format(model_directory_name) print('Reading metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] if len(class_fraction_keys) > 1: class_to_sampling_fraction_dict = dict( list(zip(class_fraction_keys, class_fraction_values))) else: class_to_sampling_fraction_dict = None training_option_dict[ trainval_io.SAMPLING_FRACTIONS_KEY] = class_to_sampling_fraction_dict example_file_names = input_examples.find_many_example_files( top_directory_name=top_example_dir_name, shuffled=False, first_spc_date_string=first_spc_date_string, last_spc_date_string=last_spc_date_string, raise_error_if_any_missing=False) training_option_dict[trainval_io.EXAMPLE_FILES_KEY] = example_file_names training_option_dict[trainval_io.FIRST_STORM_TIME_KEY] = ( time_conversion.get_start_of_spc_date(first_spc_date_string)) training_option_dict[trainval_io.LAST_STORM_TIME_KEY] = ( time_conversion.get_end_of_spc_date(last_spc_date_string)) if soundings_only: generator_object = testing_io.sounding_generator( option_dict=training_option_dict, num_examples_total=num_examples) elif model_metadata_dict[cnn.LAYER_OPERATIONS_KEY] is not None: generator_object = testing_io.gridrad_generator_2d_reduced( option_dict=training_option_dict, list_of_operation_dicts=model_metadata_dict[ cnn.LAYER_OPERATIONS_KEY], num_examples_total=num_examples) elif model_metadata_dict[cnn.CONV_2D3D_KEY]: generator_object = testing_io.myrorss_generator_2d3d( option_dict=training_option_dict, num_examples_total=num_examples) else: generator_object = testing_io.generator_2d_or_3d( option_dict=training_option_dict, num_examples_total=num_examples) include_soundings = (training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is not None) forecast_probabilities = numpy.array([]) observed_labels = numpy.array([], dtype=int) for _ in range(len(example_file_names)): try: this_storm_object_dict = next(generator_object) print(SEPARATOR_STRING) except StopIteration: break observed_labels = numpy.concatenate( (observed_labels, this_storm_object_dict[testing_io.TARGET_ARRAY_KEY])) if soundings_only: these_predictor_matrices = [ this_storm_object_dict[testing_io.SOUNDING_MATRIX_KEY] ] else: these_predictor_matrices = this_storm_object_dict[ testing_io.INPUT_MATRICES_KEY] if include_soundings: this_sounding_matrix = these_predictor_matrices[-1] else: this_sounding_matrix = None if soundings_only: this_probability_matrix = cnn.apply_cnn_soundings_only( model_object=model_object, sounding_matrix=this_sounding_matrix, verbose=True) elif model_metadata_dict[cnn.CONV_2D3D_KEY]: if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]: this_probability_matrix = cnn.apply_2d_or_3d_cnn( model_object=model_object, radar_image_matrix=these_predictor_matrices[0], sounding_matrix=this_sounding_matrix, verbose=True) else: this_probability_matrix = cnn.apply_2d3d_cnn( model_object=model_object, reflectivity_matrix_dbz=these_predictor_matrices[0], azimuthal_shear_matrix_s01=these_predictor_matrices[1], sounding_matrix=this_sounding_matrix, verbose=True) else: this_probability_matrix = cnn.apply_2d_or_3d_cnn( model_object=model_object, radar_image_matrix=these_predictor_matrices[0], sounding_matrix=this_sounding_matrix, verbose=True) print(SEPARATOR_STRING) forecast_probabilities = numpy.concatenate( (forecast_probabilities, this_probability_matrix[:, -1])) model_eval_helper.run_evaluation( forecast_probabilities=forecast_probabilities, observed_labels=observed_labels, num_bootstrap_reps=num_bootstrap_reps, confidence_level=confidence_level, output_dir_name=output_dir_name)
def _run(input_file_name, allow_whitespace, plot_significance, plot_regions_of_interest, colour_map_name, max_colour_percentile, top_output_dir_name): """Plots Grad-CAM output (class-activation maps). This is effectively the main method. :param input_file_name: See documentation at top of file. :param allow_whitespace: Same. :param plot_significance: Same. :param plot_regions_of_interest: Same. :param colour_map_name: Same. :param max_colour_percentile: Same. :param top_output_dir_name: Same. """ if plot_significance: plot_regions_of_interest = False unguided_cam_dir_name = '{0:s}/main_gradcam'.format(top_output_dir_name) guided_cam_dir_name = '{0:s}/guided_gradcam'.format(top_output_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=unguided_cam_dir_name) file_system_utils.mkdir_recursive_if_necessary( directory_name=guided_cam_dir_name) # Check input args. error_checking.assert_is_geq(max_colour_percentile, 0.) error_checking.assert_is_leq(max_colour_percentile, 100.) colour_map_object = pyplot.cm.get_cmap(colour_map_name) print('Reading data from: "{0:s}"...'.format(input_file_name)) try: gradcam_dict = gradcam.read_standard_file(input_file_name) list_of_input_matrices = gradcam_dict[gradcam.INPUT_MATRICES_KEY] list_of_cam_matrices = gradcam_dict[gradcam.CAM_MATRICES_KEY] list_of_guided_cam_matrices = gradcam_dict[ gradcam.GUIDED_CAM_MATRICES_KEY] full_storm_id_strings = gradcam_dict[gradcam.FULL_IDS_KEY] storm_times_unix_sec = gradcam_dict[gradcam.STORM_TIMES_KEY] except ValueError: gradcam_dict = gradcam.read_pmm_file(input_file_name) list_of_input_matrices = gradcam_dict[gradcam.MEAN_INPUT_MATRICES_KEY] list_of_cam_matrices = gradcam_dict[gradcam.MEAN_CAM_MATRICES_KEY] list_of_guided_cam_matrices = gradcam_dict[ gradcam.MEAN_GUIDED_CAM_MATRICES_KEY] for i in range(len(list_of_input_matrices)): list_of_input_matrices[i] = numpy.expand_dims( list_of_input_matrices[i], axis=0) if list_of_cam_matrices[i] is None: continue list_of_cam_matrices[i] = numpy.expand_dims( list_of_cam_matrices[i], axis=0) list_of_guided_cam_matrices[i] = numpy.expand_dims( list_of_guided_cam_matrices[i], axis=0) full_storm_id_strings = [None] storm_times_unix_sec = [None] pmm_flag = (full_storm_id_strings[0] is None and storm_times_unix_sec[0] is None) # Read metadata for CNN. model_file_name = gradcam_dict[gradcam.MODEL_FILE_KEY] model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0]) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) print(SEPARATOR_STRING) cam_monte_carlo_dict = (gradcam_dict[gradcam.CAM_MONTE_CARLO_KEY] if plot_significance else None) guided_cam_monte_carlo_dict = ( gradcam_dict[gradcam.GUIDED_CAM_MONTE_CARLO_KEY] if plot_significance else None) region_dict = (gradcam_dict[gradcam.REGION_DICT_KEY] if plot_regions_of_interest else None) num_examples = list_of_input_matrices[0].shape[0] num_input_matrices = len(list_of_input_matrices) for i in range(num_examples): these_figure_objects, these_axes_object_matrices = ( plot_input_examples.plot_one_example( list_of_predictor_matrices=list_of_input_matrices, model_metadata_dict=model_metadata_dict, plot_sounding=False, allow_whitespace=allow_whitespace, pmm_flag=pmm_flag, example_index=i, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i])) for j in range(num_input_matrices): if list_of_cam_matrices[j] is None: continue if cam_monte_carlo_dict is None: this_significance_matrix = None else: this_significance_matrix = numpy.logical_or( cam_monte_carlo_dict[ monte_carlo.TRIAL_PMM_MATRICES_KEY][j][i, ...] < cam_monte_carlo_dict[monte_carlo.MIN_MATRICES_KEY][j][i, ...], cam_monte_carlo_dict[ monte_carlo.TRIAL_PMM_MATRICES_KEY][j][i, ...] > cam_monte_carlo_dict[monte_carlo.MAX_MATRICES_KEY][j][i, ...]) this_num_spatial_dim = len(list_of_input_matrices[j].shape) - 2 if this_num_spatial_dim == 3: _plot_3d_radar_cam( colour_map_object=colour_map_object, max_colour_percentile=max_colour_percentile, figure_objects=these_figure_objects, axes_object_matrices=these_axes_object_matrices, model_metadata_dict=model_metadata_dict, output_dir_name=unguided_cam_dir_name, cam_matrix=list_of_cam_matrices[j][i, ...], significance_matrix=this_significance_matrix, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i]) else: if region_dict is None: _plot_2d_radar_cam( colour_map_object=colour_map_object, max_colour_percentile=max_colour_percentile, figure_objects=these_figure_objects, axes_object_matrices=these_axes_object_matrices, model_metadata_dict=model_metadata_dict, output_dir_name=unguided_cam_dir_name, cam_matrix=list_of_cam_matrices[j][i, ...], significance_matrix=this_significance_matrix, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i]) else: _plot_2d_regions( figure_objects=these_figure_objects, axes_object_matrices=these_axes_object_matrices, model_metadata_dict=model_metadata_dict, list_of_polygon_objects=region_dict[ gradcam.POLYGON_OBJECTS_KEY][j][i], output_dir_name=unguided_cam_dir_name, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i]) these_figure_objects, these_axes_object_matrices = ( plot_input_examples.plot_one_example( list_of_predictor_matrices=list_of_input_matrices, model_metadata_dict=model_metadata_dict, plot_sounding=False, allow_whitespace=allow_whitespace, pmm_flag=pmm_flag, example_index=i, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i])) for j in range(num_input_matrices): if list_of_guided_cam_matrices[j] is None: continue if guided_cam_monte_carlo_dict is None: this_significance_matrix = None else: this_significance_matrix = numpy.logical_or( guided_cam_monte_carlo_dict[ monte_carlo.TRIAL_PMM_MATRICES_KEY][j][i, ...] < guided_cam_monte_carlo_dict[monte_carlo.MIN_MATRICES_KEY] [j][i, ...], guided_cam_monte_carlo_dict[ monte_carlo.TRIAL_PMM_MATRICES_KEY][j][i, ...] > guided_cam_monte_carlo_dict[ monte_carlo.MAX_MATRICES_KEY][j][i, ...]) this_num_spatial_dim = len(list_of_input_matrices[j].shape) - 2 if this_num_spatial_dim == 3: _plot_3d_radar_cam( colour_map_object=colour_map_object, max_colour_percentile=max_colour_percentile, figure_objects=these_figure_objects, axes_object_matrices=these_axes_object_matrices, model_metadata_dict=model_metadata_dict, output_dir_name=guided_cam_dir_name, guided_cam_matrix=list_of_guided_cam_matrices[j][i, ...], significance_matrix=this_significance_matrix, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i]) else: _plot_2d_radar_cam( colour_map_object=colour_map_object, max_colour_percentile=max_colour_percentile, figure_objects=these_figure_objects, axes_object_matrices=these_axes_object_matrices, model_metadata_dict=model_metadata_dict, output_dir_name=guided_cam_dir_name, guided_cam_matrix=list_of_guided_cam_matrices[j][i, ...], significance_matrix=this_significance_matrix, full_storm_id_string=full_storm_id_strings[i], storm_time_unix_sec=storm_times_unix_sec[i])
def _run(input_file_name, saliency_colour_map_name, max_colour_prctile_for_saliency, output_dir_name): """Plots saliency maps for a CNN (convolutional neural network). This is effectively the main method. :param input_file_name: See documentation at top of file. :param saliency_colour_map_name: Same. :param max_colour_prctile_for_saliency: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) error_checking.assert_is_geq(max_colour_prctile_for_saliency, 0.) error_checking.assert_is_leq(max_colour_prctile_for_saliency, 100.) saliency_colour_map_object = pyplot.cm.get_cmap(saliency_colour_map_name) print 'Reading data from: "{0:s}"...'.format(input_file_name) try: saliency_dict = saliency_maps.read_standard_file(input_file_name) list_of_input_matrices = saliency_dict.pop( saliency_maps.INPUT_MATRICES_KEY) list_of_saliency_matrices = saliency_dict.pop( saliency_maps.SALIENCY_MATRICES_KEY) saliency_metadata_dict = saliency_dict storm_ids = saliency_metadata_dict[saliency_maps.STORM_IDS_KEY] storm_times_unix_sec = saliency_metadata_dict[ saliency_maps.STORM_TIMES_KEY] except ValueError: saliency_dict = saliency_maps.read_pmm_file(input_file_name) list_of_input_matrices = saliency_dict[ saliency_maps.MEAN_INPUT_MATRICES_KEY] list_of_saliency_matrices = saliency_dict[ saliency_maps.MEAN_SALIENCY_MATRICES_KEY] for i in range(len(list_of_input_matrices)): list_of_input_matrices[i] = numpy.expand_dims( list_of_input_matrices[i], axis=0) list_of_saliency_matrices[i] = numpy.expand_dims( list_of_saliency_matrices[i], axis=0) orig_saliency_file_name = saliency_dict[ saliency_maps.STANDARD_FILE_NAME_KEY] print 'Reading metadata from: "{0:s}"...'.format( orig_saliency_file_name) orig_saliency_dict = saliency_maps.read_standard_file( orig_saliency_file_name) orig_saliency_dict.pop(saliency_maps.INPUT_MATRICES_KEY) orig_saliency_dict.pop(saliency_maps.SALIENCY_MATRICES_KEY) saliency_metadata_dict = orig_saliency_dict storm_ids = None storm_times_unix_sec = None num_examples = list_of_input_matrices[0].shape[0] max_colour_value_by_example = numpy.full(num_examples, numpy.nan) for i in range(num_examples): these_saliency_values = numpy.concatenate( [numpy.ravel(s[i, ...]) for s in list_of_saliency_matrices] ) max_colour_value_by_example[i] = numpy.percentile( numpy.absolute(these_saliency_values), max_colour_prctile_for_saliency) model_file_name = saliency_metadata_dict[saliency_maps.MODEL_FILE_NAME_KEY] model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0] ) print 'Reading metadata from: "{0:s}"...'.format(model_metafile_name) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] sounding_field_names = training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] print SEPARATOR_STRING # if sounding_field_names is not None: # _plot_sounding_saliency( # sounding_matrix=list_of_input_matrices[-1], # sounding_saliency_matrix=list_of_saliency_matrices[-1], # sounding_field_names=sounding_field_names, # saliency_metadata_dict=saliency_metadata_dict, # colour_map_object=saliency_colour_map_object, # max_colour_value_by_example=max_colour_value_by_example, # output_dir_name=output_dir_name) # print SEPARATOR_STRING if model_metadata_dict[cnn.USE_2D3D_CONVOLUTION_KEY]: _plot_saliency_for_2d3d_radar( list_of_input_matrices=list_of_input_matrices, list_of_saliency_matrices=list_of_saliency_matrices, training_option_dict=training_option_dict, saliency_colour_map_object=saliency_colour_map_object, max_colour_value_by_example=max_colour_value_by_example, output_dir_name=output_dir_name, storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec) return num_radar_dimensions = len(list_of_input_matrices[0].shape) - 2 if num_radar_dimensions == 3: _plot_saliency_for_3d_radar( radar_matrix=list_of_input_matrices[0], radar_saliency_matrix=list_of_saliency_matrices[0], model_metadata_dict=model_metadata_dict, saliency_colour_map_object=saliency_colour_map_object, max_colour_value_by_example=max_colour_value_by_example, output_dir_name=output_dir_name, storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec) return _plot_saliency_for_2d_radar( radar_matrix=list_of_input_matrices[0], radar_saliency_matrix=list_of_saliency_matrices[0], model_metadata_dict=model_metadata_dict, saliency_colour_map_object=saliency_colour_map_object, max_colour_value_by_example=max_colour_value_by_example, output_dir_name=output_dir_name, storm_ids=storm_ids, storm_times_unix_sec=storm_times_unix_sec)
def _run(model_file_name, layer_names, top_example_dir_name, storm_metafile_name, num_examples, top_output_dir_name): """Evaluates CNN (convolutional neural net) predictions. This is effectively the main method. :param model_file_name: See documentation at top of file. :param layer_names: Same. :param top_example_dir_name: Same. :param storm_metafile_name: Same. :param num_examples: Same. :param top_output_dir_name: Same. :raises: ValueError: if feature maps do not have 2 or 3 spatial dimensions. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = cnn.read_model(model_file_name) model_metafile_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0]) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = cnn.read_model_metadata(model_metafile_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None print( 'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name)) full_id_strings, storm_times_unix_sec = tracking_io.read_ids_and_times( storm_metafile_name) print(SEPARATOR_STRING) if 0 < num_examples < len(full_id_strings): full_id_strings = full_id_strings[:num_examples] storm_times_unix_sec = storm_times_unix_sec[:num_examples] list_of_predictor_matrices = testing_io.read_specific_examples( top_example_dir_name=top_example_dir_name, desired_full_id_strings=full_id_strings, desired_times_unix_sec=storm_times_unix_sec, option_dict=training_option_dict, list_of_layer_operation_dicts=model_metadata_dict[ cnn.LAYER_OPERATIONS_KEY])[0] print(SEPARATOR_STRING) include_soundings = (training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is not None) if include_soundings: sounding_matrix = list_of_predictor_matrices[-1] else: sounding_matrix = None num_layers = len(layer_names) feature_matrix_by_layer = [None] * num_layers for k in range(num_layers): if model_metadata_dict[cnn.CONV_2D3D_KEY]: if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]: feature_matrix_by_layer[k] = cnn.apply_2d_or_3d_cnn( model_object=model_object, radar_image_matrix=list_of_predictor_matrices[0], sounding_matrix=sounding_matrix, return_features=True, feature_layer_name=layer_names[k]) else: feature_matrix_by_layer[k] = cnn.apply_2d3d_cnn( model_object=model_object, reflectivity_matrix_dbz=list_of_predictor_matrices[0], azimuthal_shear_matrix_s01=list_of_predictor_matrices[1], sounding_matrix=sounding_matrix, return_features=True, feature_layer_name=layer_names[k]) else: feature_matrix_by_layer[k] = cnn.apply_2d_or_3d_cnn( model_object=model_object, radar_image_matrix=list_of_predictor_matrices[0], sounding_matrix=sounding_matrix, return_features=True, feature_layer_name=layer_names[k]) for k in range(num_layers): this_output_dir_name = '{0:s}/{1:s}'.format(top_output_dir_name, layer_names[k]) file_system_utils.mkdir_recursive_if_necessary( directory_name=this_output_dir_name) _plot_feature_maps_one_layer(feature_matrix=feature_matrix_by_layer[k], full_id_strings=full_id_strings, storm_times_unix_sec=storm_times_unix_sec, layer_name=layer_names[k], output_dir_name=this_output_dir_name) print(SEPARATOR_STRING)
def _read_target_values(top_target_dir_name, storm_activations, activation_metadata_dict): """Reads target value for each storm object. E = number of examples (storm objects) :param top_target_dir_name: See documentation at top of file. :param storm_activations: length-E numpy array of activations. :param activation_metadata_dict: Dictionary returned by `model_activation.read_file`. :return: target_dict: Dictionary with the following keys. target_dict['full_id_strings']: length-E list of full storm IDs. target_dict['storm_times_unix_sec']: length-E numpy array of storm times. target_dict['storm_activations']: length-E numpy array of model activations. target_dict['storm_target_values']: length-E numpy array of target values. :raises: ValueError: if the target variable is multiclass and not binarized. """ # Convert input args. full_id_strings = activation_metadata_dict[model_activation.FULL_IDS_KEY] storm_times_unix_sec = activation_metadata_dict[ model_activation.STORM_TIMES_KEY] storm_spc_date_strings_numpy = numpy.array([ time_conversion.time_to_spc_date_string(t) for t in storm_times_unix_sec ], dtype=object) unique_spc_date_strings_numpy = numpy.unique(storm_spc_date_strings_numpy) # Read metadata for machine-learning model. model_file_name = activation_metadata_dict[ model_activation.MODEL_FILE_NAME_KEY] model_metadata_file_name = '{0:s}/model_metadata.p'.format( os.path.split(model_file_name)[0]) print('Reading metadata from: "{0:s}"...'.format(model_metadata_file_name)) model_metadata_dict = cnn.read_model_metadata(model_metadata_file_name) training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] target_name = training_option_dict[trainval_io.TARGET_NAME_KEY] num_classes = target_val_utils.target_name_to_num_classes( target_name=target_name, include_dead_storms=False) binarize_target = (training_option_dict[trainval_io.BINARIZE_TARGET_KEY] and num_classes > 2) if num_classes > 2 and not binarize_target: error_string = ( 'The target variable ("{0:s}") is multiclass, which this script ' 'cannot handle.').format(target_name) raise ValueError(error_string) event_type_string = target_val_utils.target_name_to_params(target_name)[ target_val_utils.EVENT_TYPE_KEY] # Read target values. storm_target_values = numpy.array([], dtype=int) id_sort_indices = numpy.array([], dtype=int) num_spc_dates = len(unique_spc_date_strings_numpy) for i in range(num_spc_dates): this_target_file_name = target_val_utils.find_target_file( top_directory_name=top_target_dir_name, event_type_string=event_type_string, spc_date_string=unique_spc_date_strings_numpy[i]) print('Reading data from: "{0:s}"...'.format(this_target_file_name)) this_target_value_dict = target_val_utils.read_target_values( netcdf_file_name=this_target_file_name, target_names=[target_name]) these_indices = numpy.where(storm_spc_date_strings_numpy == unique_spc_date_strings_numpy[i])[0] id_sort_indices = numpy.concatenate((id_sort_indices, these_indices)) these_indices = tracking_utils.find_storm_objects( all_id_strings=this_target_value_dict[ target_val_utils.FULL_IDS_KEY], all_times_unix_sec=this_target_value_dict[ target_val_utils.VALID_TIMES_KEY], id_strings_to_keep=[full_id_strings[k] for k in these_indices], times_to_keep_unix_sec=storm_times_unix_sec[these_indices]) if len(these_indices) == 0: continue these_target_values = this_target_value_dict[ target_val_utils.TARGET_MATRIX_KEY][these_indices, :] these_target_values = numpy.reshape(these_target_values, these_target_values.size) storm_target_values = numpy.concatenate( (storm_target_values, these_target_values)) good_indices = numpy.where( storm_target_values != target_val_utils.INVALID_STORM_INTEGER)[0] storm_target_values = storm_target_values[good_indices] id_sort_indices = id_sort_indices[good_indices] if binarize_target: storm_target_values = (storm_target_values == num_classes - 1).astype(int) return { FULL_IDS_KEY: [full_id_strings[k] for k in id_sort_indices], STORM_TIMES_KEY: storm_times_unix_sec[id_sort_indices], STORM_ACTIVATIONS_KEY: storm_activations[id_sort_indices], TARGET_VALUES_KEY: storm_target_values }