def _run(saliency_file_name, num_examples_to_plot, colour_map_name, max_colour_percentile, output_dir_name): """Plots saliency maps for all target variables. This is effectively the main method. :param saliency_file_name: See documentation at top of file. :param num_examples_to_plot: Same. :param colour_map_name: Same. :param max_colour_percentile: Same. :param output_dir_name: Same. """ colour_map_object = pyplot.get_cmap(colour_map_name) error_checking.assert_is_geq(max_colour_percentile, 90.) error_checking.assert_is_leq(max_colour_percentile, 100.) file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) print( 'Reading saliency values from: "{0:s}"...'.format(saliency_file_name)) saliency_dict = saliency.read_all_targets_file(saliency_file_name) model_file_name = saliency_dict[saliency.MODEL_FILE_KEY] model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0]) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) num_examples_total = len(saliency_dict[saliency.EXAMPLE_IDS_KEY]) if num_examples_to_plot <= 0: num_examples_to_plot = num_examples_total num_examples_to_plot = numpy.minimum(num_examples_to_plot, num_examples_total) print(SEPARATOR_STRING) for i in range(num_examples_to_plot): _plot_saliency_one_example(saliency_dict=saliency_dict, example_index=i, model_metadata_dict=model_metadata_dict, colour_map_object=colour_map_object, max_colour_percentile=max_colour_percentile, output_dir_name=output_dir_name) print(SEPARATOR_STRING)
def _run(gradcam_file_name, colour_map_name, max_colour_percentile, output_dir_name): """Plots Grad-CAM output (class-activation maps) for all target variables. This is effectively the main method. :param gradcam_file_name: See documentation at top of file. :param colour_map_name: Same. :param max_colour_percentile: Same. :param output_dir_name: Same. """ colour_map_object = pyplot.get_cmap(colour_map_name) error_checking.assert_is_geq(max_colour_percentile, 90.) error_checking.assert_is_leq(max_colour_percentile, 100.) file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name ) print('Reading class-activation maps from: "{0:s}"...'.format( gradcam_file_name )) gradcam_dict = gradcam.read_all_targets_file(gradcam_file_name) model_file_name = gradcam_dict[gradcam.MODEL_FILE_KEY] model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0] ) print('Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) num_examples = len(gradcam_dict[gradcam.EXAMPLE_IDS_KEY]) print(SEPARATOR_STRING) for i in range(num_examples): _plot_gradcam_one_example( gradcam_dict=gradcam_dict, example_index=i, model_metadata_dict=model_metadata_dict, colour_map_object=colour_map_object, max_colour_percentile=max_colour_percentile, output_dir_name=output_dir_name ) print(SEPARATOR_STRING)
def _run(input_file_name, use_log_scale, output_dir_name): """Plots results of backwards optimization. This is effectively the main method. :param input_file_name: See documentation at top of file. :param use_log_scale: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) print('Reading backwards-optimization results from: "{0:s}"...'.format( input_file_name)) bwo_dict = bwo.read_file(input_file_name) model_file_name = bwo_dict[bwo.MODEL_FILE_KEY] model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0]) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) print(SEPARATOR_STRING) num_examples = len(bwo_dict[bwo.EXAMPLE_IDS_KEY]) for i in range(num_examples): _plot_results_one_example(bwo_dict=bwo_dict, example_index=i, model_metadata_dict=model_metadata_dict, use_log_scale=use_log_scale, output_dir_name=output_dir_name) if i != num_examples - 1: print(MINOR_SEPARATOR_STRING)
def _run(evaluation_file_names, line_styles, line_colour_strings, set_descriptions_verbose, confidence_level, use_log_scale, plot_by_height, output_dir_name): """Plots model evaluation. This is effectively the main method. :param evaluation_file_names: See documentation at top of file. :param line_styles: Same. :param line_colour_strings: Same. :param set_descriptions_verbose: Same. :param confidence_level: Same. :param use_log_scale: Same. :param plot_by_height: Same. :param output_dir_name: Same. """ # Check input args. file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) if confidence_level < 0: confidence_level = None if confidence_level is not None: error_checking.assert_is_geq(confidence_level, 0.9) error_checking.assert_is_less_than(confidence_level, 1.) num_evaluation_sets = len(evaluation_file_names) expected_dim = numpy.array([num_evaluation_sets], dtype=int) error_checking.assert_is_string_list(line_styles) error_checking.assert_is_numpy_array(numpy.array(line_styles), exact_dimensions=expected_dim) error_checking.assert_is_string_list(set_descriptions_verbose) error_checking.assert_is_numpy_array(numpy.array(set_descriptions_verbose), exact_dimensions=expected_dim) set_descriptions_verbose = [ s.replace('_', ' ') for s in set_descriptions_verbose ] set_descriptions_abbrev = [ s.lower().replace(' ', '-') for s in set_descriptions_verbose ] error_checking.assert_is_string_list(line_colour_strings) error_checking.assert_is_numpy_array(numpy.array(line_colour_strings), exact_dimensions=expected_dim) line_colours = [ numpy.fromstring(s, dtype=float, sep='_') / 255 for s in line_colour_strings ] for i in range(num_evaluation_sets): error_checking.assert_is_numpy_array(line_colours[i], exact_dimensions=numpy.array( [3], dtype=int)) error_checking.assert_is_geq_numpy_array(line_colours[i], 0.) error_checking.assert_is_leq_numpy_array(line_colours[i], 1.) # Read files. evaluation_tables_xarray = [xarray.Dataset()] * num_evaluation_sets prediction_dicts = [dict()] * num_evaluation_sets for i in range(num_evaluation_sets): print('Reading data from: "{0:s}"...'.format(evaluation_file_names[i])) evaluation_tables_xarray[i] = evaluation.read_file( evaluation_file_names[i]) this_prediction_file_name = ( evaluation_tables_xarray[i].attrs[evaluation.PREDICTION_FILE_KEY]) print( 'Reading data from: "{0:s}"...'.format(this_prediction_file_name)) prediction_dicts[i] = prediction_io.read_file( this_prediction_file_name) model_file_name = ( evaluation_tables_xarray[0].attrs[evaluation.MODEL_FILE_KEY]) model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True) print('Reading metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) generator_option_dict = model_metadata_dict[ neural_net.TRAINING_OPTIONS_KEY] scalar_target_names = ( generator_option_dict[neural_net.SCALAR_TARGET_NAMES_KEY]) vector_target_names = ( generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY]) heights_m_agl = generator_option_dict[neural_net.HEIGHTS_KEY] try: t = evaluation_tables_xarray[0] aux_target_names = t.coords[evaluation.AUX_TARGET_FIELD_DIM].values except: aux_target_names = [] num_scalar_targets = len(scalar_target_names) num_vector_targets = len(vector_target_names) num_heights = len(heights_m_agl) num_aux_targets = len(aux_target_names) example_dict = { example_utils.SCALAR_TARGET_NAMES_KEY: scalar_target_names, example_utils.VECTOR_TARGET_NAMES_KEY: vector_target_names, example_utils.HEIGHTS_KEY: heights_m_agl, example_utils.SCALAR_PREDICTOR_NAMES_KEY: generator_option_dict[neural_net.SCALAR_PREDICTOR_NAMES_KEY], example_utils.VECTOR_PREDICTOR_NAMES_KEY: generator_option_dict[neural_net.VECTOR_PREDICTOR_NAMES_KEY] } normalization_file_name = ( generator_option_dict[neural_net.NORMALIZATION_FILE_KEY]) print(('Reading training examples (for climatology) from: "{0:s}"...' ).format(normalization_file_name)) training_example_dict = example_io.read_file(normalization_file_name) training_example_dict = example_utils.subset_by_height( example_dict=training_example_dict, heights_m_agl=heights_m_agl) mean_training_example_dict = normalization.create_mean_example( new_example_dict=example_dict, training_example_dict=training_example_dict) print(SEPARATOR_STRING) # Do actual stuff. _plot_error_distributions( prediction_dicts=prediction_dicts, model_metadata_dict=model_metadata_dict, aux_target_names=aux_target_names, set_descriptions_abbrev=set_descriptions_abbrev, set_descriptions_verbose=set_descriptions_verbose, output_dir_name=output_dir_name) print(SEPARATOR_STRING) _plot_reliability_by_height( evaluation_tables_xarray=evaluation_tables_xarray, vector_target_names=vector_target_names, heights_m_agl=heights_m_agl, set_descriptions_abbrev=set_descriptions_abbrev, set_descriptions_verbose=set_descriptions_verbose, output_dir_name=output_dir_name) print(SEPARATOR_STRING) for k in range(num_vector_targets): for this_score_name in list(SCORE_NAME_TO_PROFILE_KEY.keys()): _plot_score_profile( evaluation_tables_xarray=evaluation_tables_xarray, line_styles=line_styles, line_colours=line_colours, set_descriptions_verbose=set_descriptions_verbose, confidence_level=confidence_level, target_name=vector_target_names[k], score_name=this_score_name, use_log_scale=use_log_scale, output_dir_name=output_dir_name) print(SEPARATOR_STRING) for k in range(num_scalar_targets): _plot_attributes_diagram( evaluation_tables_xarray=evaluation_tables_xarray, line_styles=line_styles, line_colours=line_colours, set_descriptions_abbrev=set_descriptions_abbrev, set_descriptions_verbose=set_descriptions_verbose, confidence_level=confidence_level, mean_training_example_dict=mean_training_example_dict, target_name=scalar_target_names[k], output_dir_name=output_dir_name) for k in range(num_aux_targets): _plot_attributes_diagram( evaluation_tables_xarray=evaluation_tables_xarray, line_styles=line_styles, line_colours=line_colours, set_descriptions_abbrev=set_descriptions_abbrev, set_descriptions_verbose=set_descriptions_verbose, confidence_level=confidence_level, mean_training_example_dict=mean_training_example_dict, target_name=aux_target_names[k], output_dir_name=output_dir_name) if not plot_by_height: return print(SEPARATOR_STRING) for k in range(num_vector_targets): for j in range(num_heights): _plot_attributes_diagram( evaluation_tables_xarray=evaluation_tables_xarray, line_styles=line_styles, line_colours=line_colours, set_descriptions_abbrev=set_descriptions_abbrev, set_descriptions_verbose=set_descriptions_verbose, confidence_level=confidence_level, mean_training_example_dict=mean_training_example_dict, height_m_agl=heights_m_agl[j], target_name=vector_target_names[k], output_dir_name=output_dir_name) if k != num_vector_targets - 1: print(SEPARATOR_STRING)
def _run(model_file_name, example_file_name, num_examples, example_dir_name, example_id_file_name, ideal_activation, scalar_output_layer_name, vector_output_layer_name, output_file_name): """Makes saliency map for each example and target, according to one model. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_file_name: Same. :param num_examples: Same. :param example_dir_name: Same. :param example_id_file_name: Same. :param ideal_activation: Same. :param scalar_output_layer_name: Same. :param vector_output_layer_name: Same. :param output_file_name: Same. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True ) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) predictor_matrix, _, example_id_strings = ( misc_utils.get_examples_for_inference( model_metadata_dict=metadata_dict, example_file_name=example_file_name, num_examples=num_examples, example_dir_name=example_dir_name, example_id_file_name=example_id_file_name ) ) print(SEPARATOR_STRING) net_type_string = metadata_dict[neural_net.NET_TYPE_KEY] generator_option_dict = metadata_dict[neural_net.TRAINING_OPTIONS_KEY] scalar_predictor_names = ( generator_option_dict[neural_net.SCALAR_PREDICTOR_NAMES_KEY] ) vector_predictor_names = ( generator_option_dict[neural_net.VECTOR_PREDICTOR_NAMES_KEY] ) scalar_target_names = ( generator_option_dict[neural_net.SCALAR_TARGET_NAMES_KEY] ) vector_target_names = ( generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY] ) heights_m_agl = generator_option_dict[neural_net.HEIGHTS_KEY] num_scalar_predictors = len(scalar_predictor_names) num_vector_predictors = len(vector_predictor_names) num_scalar_targets = len(scalar_target_names) num_vector_targets = len(vector_target_names) num_heights = len(heights_m_agl) num_examples = len(example_id_strings) if net_type_string == neural_net.DENSE_NET_TYPE_STRING: saliency_matrix_scalar_p_scalar_t = numpy.full( (num_examples, num_scalar_predictors, num_scalar_targets), numpy.nan ) else: saliency_matrix_scalar_p_scalar_t = numpy.full( (num_examples, num_heights, num_scalar_predictors, num_scalar_targets), numpy.nan ) saliency_matrix_vector_p_scalar_t = numpy.full( (num_examples, num_heights, num_vector_predictors, num_scalar_targets), numpy.nan ) if net_type_string == neural_net.DENSE_NET_TYPE_STRING: saliency_matrix_scalar_p_vector_t = numpy.full( (num_examples, num_scalar_predictors, num_heights, num_vector_targets), numpy.nan ) else: saliency_matrix_scalar_p_vector_t = numpy.full( (num_examples, num_heights, num_scalar_predictors, num_heights, num_vector_targets), numpy.nan ) saliency_matrix_vector_p_vector_t = numpy.full( (num_examples, num_heights, num_vector_predictors, num_heights, num_vector_targets), numpy.nan ) dummy_example_dict = { example_utils.SCALAR_PREDICTOR_NAMES_KEY: scalar_predictor_names, example_utils.VECTOR_PREDICTOR_NAMES_KEY: vector_predictor_names, example_utils.SCALAR_TARGET_NAMES_KEY: scalar_target_names, example_utils.VECTOR_TARGET_NAMES_KEY: vector_target_names, example_utils.HEIGHTS_KEY: heights_m_agl } for k in range(num_scalar_targets): these_neuron_indices = neural_net.target_var_to_neuron_indices( example_dict=copy.deepcopy(dummy_example_dict), net_type_string=net_type_string, target_name=scalar_target_names[k] ) print('Computing saliency for "{0:s}"...'.format( scalar_target_names[k] )) this_saliency_matrix = saliency.get_saliency_one_neuron( model_object=model_object, predictor_matrix=predictor_matrix, layer_name=scalar_output_layer_name, neuron_indices=these_neuron_indices, ideal_activation=ideal_activation ) if net_type_string == neural_net.DENSE_NET_TYPE_STRING: new_example_dict = neural_net.predictors_numpy_to_dict( predictor_matrix=this_saliency_matrix, example_dict=copy.deepcopy(dummy_example_dict), net_type_string=net_type_string ) saliency_matrix_scalar_p_scalar_t[..., k] = ( new_example_dict[example_utils.SCALAR_PREDICTOR_VALS_KEY] ) saliency_matrix_vector_p_scalar_t[..., k] = ( new_example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] ) else: saliency_matrix_scalar_p_scalar_t[..., k] = ( this_saliency_matrix[..., -num_scalar_predictors:] ) saliency_matrix_vector_p_scalar_t[..., k] = ( this_saliency_matrix[..., :-num_scalar_predictors] ) print(SEPARATOR_STRING) for k in range(num_vector_targets): for j in range(num_heights): these_neuron_indices = neural_net.target_var_to_neuron_indices( example_dict=copy.deepcopy(dummy_example_dict), net_type_string=net_type_string, target_name=vector_target_names[k], height_m_agl=heights_m_agl[j] ) print('Computing saliency for "{0:s}" at {1:d} m AGL...'.format( vector_target_names[k], int(numpy.round(heights_m_agl[j])) )) this_saliency_matrix = saliency.get_saliency_one_neuron( model_object=model_object, predictor_matrix=predictor_matrix, layer_name=vector_output_layer_name, neuron_indices=these_neuron_indices, ideal_activation=ideal_activation ) if net_type_string == neural_net.DENSE_NET_TYPE_STRING: new_example_dict = neural_net.predictors_numpy_to_dict( predictor_matrix=this_saliency_matrix, example_dict=copy.deepcopy(dummy_example_dict), net_type_string=net_type_string ) saliency_matrix_scalar_p_vector_t[..., j, k] = ( new_example_dict[example_utils.SCALAR_PREDICTOR_VALS_KEY] ) saliency_matrix_vector_p_vector_t[..., j, k] = ( new_example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] ) else: saliency_matrix_scalar_p_vector_t[..., j, k] = ( this_saliency_matrix[..., -num_scalar_predictors:] ) saliency_matrix_vector_p_vector_t[..., j, k] = ( this_saliency_matrix[..., :-num_scalar_predictors] ) print(SEPARATOR_STRING) print('Writing saliency maps to: "{0:s}"...'.format(output_file_name)) saliency.write_all_targets_file( netcdf_file_name=output_file_name, saliency_matrix_scalar_p_scalar_t=saliency_matrix_scalar_p_scalar_t, saliency_matrix_vector_p_scalar_t=saliency_matrix_vector_p_scalar_t, saliency_matrix_scalar_p_vector_t=saliency_matrix_scalar_p_vector_t, saliency_matrix_vector_p_vector_t=saliency_matrix_vector_p_vector_t, example_id_strings=example_id_strings, model_file_name=model_file_name, ideal_activation=ideal_activation )
def _run(model_file_name, layer_names, example_file_name, num_examples, example_dir_name, example_id_file_name, output_dir_name): """Plots feature maps for each layer/example pair, for a single neural net. This is effectively the main method. :param model_file_name: See documentation at top of file. :param layer_names: Same. :param example_file_name: Same. :param num_examples: Same. :param example_dir_name: Same. :param example_id_file_name: Same. :param output_dir_name: Same. :raises: ValueError: if neural-net type is not CNN or U-net. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True ) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) net_type_string = metadata_dict[neural_net.NET_TYPE_KEY] valid_net_type_strings = [ neural_net.CNN_TYPE_STRING, neural_net.U_NET_TYPE_STRING ] if net_type_string not in valid_net_type_strings: error_string = ( '\nThis script does not work for net type "{0:s}". Works only for ' 'those listed below:\n{1:s}' ).format(net_type_string, str(valid_net_type_strings)) raise ValueError(error_string) predictor_matrix, _, example_id_strings = ( misc_utils.get_examples_for_inference( model_metadata_dict=metadata_dict, example_file_name=example_file_name, num_examples=num_examples, example_dir_name=example_dir_name, example_id_file_name=example_id_file_name ) ) print(SEPARATOR_STRING) num_layers = len(layer_names) feature_matrix_by_layer = [numpy.array([])] * num_layers for k in range(num_layers): print('Creating feature maps for layer "{0:s}"...'.format( layer_names[k] )) feature_matrix_by_layer[k] = neural_net.get_feature_maps( model_object=model_object, predictor_matrix=predictor_matrix, num_examples_per_batch=predictor_matrix.shape[0], feature_layer_name=layer_names[k], verbose=False ) print('\n') for k in range(num_layers): this_output_dir_name = '{0:s}/{1:s}'.format( output_dir_name, layer_names[k] ) file_system_utils.mkdir_recursive_if_necessary( directory_name=this_output_dir_name ) _plot_feature_maps_one_layer( feature_matrix=feature_matrix_by_layer[k], example_id_strings=example_id_strings, layer_name=layer_names[k], output_dir_name=this_output_dir_name ) print(SEPARATOR_STRING)
def read_all_targets_file(netcdf_file_name): """Reads saliency maps for all target variables from NetCDF file. :param netcdf_file_name: Path to input file. :return: saliency_dict: Dictionary with the following keys. saliency_dict['saliency_matrix_scalar_p_scalar_t']: See doc for `write_all_targets_file`. saliency_dict['saliency_matrix_vector_p_scalar_t']: Same. saliency_dict['saliency_matrix_scalar_p_vector_t']: Same. saliency_dict['saliency_matrix_vector_p_vector_t']: Same. saliency_dict['example_id_strings']: Same. saliency_dict['model_file_name']: Same. saliency_dict['ideal_activation']: Same. """ dataset_object = netCDF4.Dataset(netcdf_file_name) saliency_dict = { EXAMPLE_IDS_KEY: [ str(id) for id in netCDF4.chartostring( dataset_object.variables[EXAMPLE_IDS_KEY][:]) ], MODEL_FILE_KEY: str(getattr(dataset_object, MODEL_FILE_KEY)), IDEAL_ACTIVATION_KEY: getattr(dataset_object, IDEAL_ACTIVATION_KEY) } num_examples = dataset_object.dimensions[EXAMPLE_DIMENSION_KEY].size num_scalar_predictors = ( dataset_object.dimensions[SCALAR_PREDICTOR_DIM_KEY].size) num_vector_predictors = ( dataset_object.dimensions[VECTOR_PREDICTOR_DIM_KEY].size) num_heights = dataset_object.dimensions[HEIGHT_DIMENSION_KEY].size num_scalar_targets = dataset_object.dimensions[SCALAR_TARGET_DIM_KEY].size num_vector_targets = dataset_object.dimensions[VECTOR_TARGET_DIM_KEY].size if SALIENCY_SCALAR_P_SCALAR_T_KEY in dataset_object.variables: saliency_dict[SALIENCY_SCALAR_P_SCALAR_T_KEY] = ( dataset_object.variables[SALIENCY_SCALAR_P_SCALAR_T_KEY][:]) else: model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(saliency_dict[MODEL_FILE_KEY])[0]) model_metadata_dict = neural_net.read_metafile(model_metafile_name) net_type_string = model_metadata_dict[neural_net.NET_TYPE_KEY] if net_type_string == neural_net.DENSE_NET_TYPE_STRING: these_dim = (num_examples, num_scalar_predictors, num_scalar_targets) else: these_dim = (num_examples, num_heights, num_scalar_predictors, num_scalar_targets) saliency_dict[SALIENCY_SCALAR_P_SCALAR_T_KEY] = numpy.full( these_dim, 0.) if SALIENCY_VECTOR_P_SCALAR_T_KEY in dataset_object.variables: saliency_dict[SALIENCY_VECTOR_P_SCALAR_T_KEY] = ( dataset_object.variables[SALIENCY_VECTOR_P_SCALAR_T_KEY][:]) else: these_dim = (num_examples, num_heights, num_vector_predictors, num_scalar_targets) saliency_dict[SALIENCY_VECTOR_P_SCALAR_T_KEY] = numpy.full( these_dim, 0.) if SALIENCY_SCALAR_P_VECTOR_T_KEY in dataset_object.variables: saliency_dict[SALIENCY_SCALAR_P_VECTOR_T_KEY] = ( dataset_object.variables[SALIENCY_SCALAR_P_VECTOR_T_KEY][:]) else: model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(saliency_dict[MODEL_FILE_KEY])[0]) model_metadata_dict = neural_net.read_metafile(model_metafile_name) net_type_string = model_metadata_dict[neural_net.NET_TYPE_KEY] if net_type_string == neural_net.DENSE_NET_TYPE_STRING: these_dim = (num_examples, num_scalar_predictors, num_heights, num_vector_targets) else: these_dim = (num_examples, num_heights, num_scalar_predictors, num_heights, num_vector_targets) saliency_dict[SALIENCY_SCALAR_P_VECTOR_T_KEY] = numpy.full( these_dim, 0.) if SALIENCY_VECTOR_P_VECTOR_T_KEY in dataset_object.variables: saliency_dict[SALIENCY_VECTOR_P_VECTOR_T_KEY] = ( dataset_object.variables[SALIENCY_VECTOR_P_VECTOR_T_KEY][:]) else: these_dim = (num_examples, num_heights, num_vector_predictors, num_heights, num_vector_targets) saliency_dict[SALIENCY_VECTOR_P_VECTOR_T_KEY] = numpy.full( these_dim, 0.) if numpy.isnan(saliency_dict[IDEAL_ACTIVATION_KEY]): saliency_dict[IDEAL_ACTIVATION_KEY] = None dataset_object.close() return saliency_dict
def _run(model_file_name, example_file_name, num_examples, example_dir_name, example_id_file_name, layer_name, is_layer_output, neuron_indices, ideal_activation, output_file_name): """Makes saliency map for each example, according to one model. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_file_name: Same. :param num_examples: Same. :param example_dir_name: Same. :param example_id_file_name: Same. :param layer_name: Same. :param is_layer_output: Same. :param neuron_indices: Same. :param ideal_activation: Same. :param output_file_name: Same. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True ) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) predictor_matrix, _, example_id_strings = ( misc_utils.get_examples_for_inference( model_metadata_dict=metadata_dict, example_file_name=example_file_name, num_examples=num_examples, example_dir_name=example_dir_name, example_id_file_name=example_id_file_name ) ) print(SEPARATOR_STRING) generator_option_dict = metadata_dict[neural_net.TRAINING_OPTIONS_KEY] if is_layer_output: dummy_example_dict = { example_utils.SCALAR_TARGET_NAMES_KEY: generator_option_dict[neural_net.SCALAR_TARGET_NAMES_KEY], example_utils.VECTOR_TARGET_NAMES_KEY: generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY], example_utils.HEIGHTS_KEY: generator_option_dict[neural_net.HEIGHTS_KEY] } target_field_name, target_height_m_agl = ( neural_net.neuron_indices_to_target_var( neuron_indices=neuron_indices, example_dict=copy.deepcopy(dummy_example_dict), net_type_string=metadata_dict[neural_net.NET_TYPE_KEY] ) ) else: target_field_name = None target_height_m_agl = None print('Target field and height = {0:s}, {1:s}'.format( str(target_field_name), str(target_height_m_agl) )) print('Computing saliency for neuron {0:s} in layer "{1:s}"...'.format( str(neuron_indices), layer_name )) saliency_matrix = saliency.get_saliency_one_neuron( model_object=model_object, predictor_matrix=predictor_matrix, layer_name=layer_name, neuron_indices=neuron_indices, ideal_activation=ideal_activation ) net_type_string = metadata_dict[neural_net.NET_TYPE_KEY] if net_type_string == neural_net.DENSE_NET_TYPE_STRING: dummy_example_dict = { example_utils.SCALAR_PREDICTOR_NAMES_KEY: generator_option_dict[neural_net.SCALAR_PREDICTOR_NAMES_KEY], example_utils.VECTOR_PREDICTOR_NAMES_KEY: generator_option_dict[neural_net.VECTOR_PREDICTOR_NAMES_KEY], example_utils.HEIGHTS_KEY: generator_option_dict[neural_net.HEIGHTS_KEY] } dummy_example_dict = neural_net.predictors_numpy_to_dict( predictor_matrix=saliency_matrix, example_dict=dummy_example_dict, net_type_string=metadata_dict[neural_net.NET_TYPE_KEY] ) scalar_saliency_matrix = ( dummy_example_dict[example_utils.SCALAR_PREDICTOR_VALS_KEY] ) vector_saliency_matrix = ( dummy_example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] ) else: num_scalar_predictors = len( generator_option_dict[neural_net.SCALAR_PREDICTOR_NAMES_KEY] ) scalar_saliency_matrix = saliency_matrix[..., -num_scalar_predictors:] vector_saliency_matrix = saliency_matrix[..., :-num_scalar_predictors] print('Writing saliency maps to: "{0:s}"...'.format(output_file_name)) saliency.write_file( netcdf_file_name=output_file_name, scalar_saliency_matrix=scalar_saliency_matrix, vector_saliency_matrix=vector_saliency_matrix, example_id_strings=example_id_strings, model_file_name=model_file_name, layer_name=layer_name, neuron_indices=neuron_indices, ideal_activation=ideal_activation, target_field_name=target_field_name, target_height_m_agl=target_height_m_agl )
def _run(model_file_name, example_dir_name, first_time_string, last_time_string, exclude_summit_greenland, output_file_name): """Applies trained neural net in inference mode. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_dir_name: Same. :param first_time_string: Same. :param last_time_string: Same. :param exclude_summit_greenland: Same. :param output_file_name: Same. """ first_time_unix_sec = time_conversion.string_to_unix_sec( first_time_string, TIME_FORMAT) last_time_unix_sec = time_conversion.string_to_unix_sec( last_time_string, TIME_FORMAT) print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) generator_option_dict = copy.deepcopy( metadata_dict[neural_net.TRAINING_OPTIONS_KEY]) generator_option_dict[neural_net.EXAMPLE_DIRECTORY_KEY] = example_dir_name generator_option_dict[neural_net.FIRST_TIME_KEY] = first_time_unix_sec generator_option_dict[neural_net.LAST_TIME_KEY] = last_time_unix_sec vector_target_norm_type_string = copy.deepcopy( generator_option_dict[neural_net.VECTOR_TARGET_NORM_TYPE_KEY]) scalar_target_norm_type_string = copy.deepcopy( generator_option_dict[neural_net.SCALAR_TARGET_NORM_TYPE_KEY]) generator_option_dict[neural_net.VECTOR_TARGET_NORM_TYPE_KEY] = None generator_option_dict[neural_net.SCALAR_TARGET_NORM_TYPE_KEY] = None net_type_string = metadata_dict[neural_net.NET_TYPE_KEY] predictor_matrix, target_array, example_id_strings = neural_net.create_data( option_dict=generator_option_dict, for_inference=True, net_type_string=net_type_string, exclude_summit_greenland=exclude_summit_greenland) print(SEPARATOR_STRING) exec_start_time_unix_sec = time.time() prediction_array = neural_net.apply_model( model_object=model_object, predictor_matrix=predictor_matrix, num_examples_per_batch=NUM_EXAMPLES_PER_BATCH, net_type_string=net_type_string, verbose=True) print(SEPARATOR_STRING) print('Time to apply neural net = {0:.4f} seconds'.format( time.time() - exec_start_time_unix_sec)) vector_target_matrix = target_array[0] vector_prediction_matrix = prediction_array[0] if len(target_array) == 2: scalar_target_matrix = target_array[1] scalar_prediction_matrix = prediction_array[1] else: scalar_target_matrix = None scalar_prediction_matrix = None target_example_dict = _targets_numpy_to_dict( scalar_target_matrix=scalar_target_matrix, vector_target_matrix=vector_target_matrix, model_metadata_dict=metadata_dict) prediction_example_dict = _targets_numpy_to_dict( scalar_target_matrix=scalar_prediction_matrix, vector_target_matrix=vector_prediction_matrix, model_metadata_dict=metadata_dict) normalization_file_name = ( generator_option_dict[neural_net.NORMALIZATION_FILE_KEY]) print(('Reading training examples (for normalization) from: "{0:s}"...' ).format(normalization_file_name)) training_example_dict = example_io.read_file(normalization_file_name) training_example_dict = example_utils.subset_by_height( example_dict=training_example_dict, heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY]) num_examples = len(example_id_strings) num_heights = len(prediction_example_dict[example_utils.HEIGHTS_KEY]) this_dict = { example_utils.VECTOR_PREDICTOR_NAMES_KEY: [], example_utils.VECTOR_PREDICTOR_VALS_KEY: numpy.full((num_examples, num_heights, 0), 0.), example_utils.SCALAR_PREDICTOR_NAMES_KEY: [], example_utils.SCALAR_PREDICTOR_VALS_KEY: numpy.full((num_examples, 0), 0.) } target_example_dict.update(this_dict) prediction_example_dict.update(this_dict) if vector_target_norm_type_string is not None: print('Denormalizing predicted vectors...') # down_flux_inc_matrix_w_m03 = example_utils.get_field_from_dict( # example_dict=prediction_example_dict, # field_name=example_utils.SHORTWAVE_DOWN_FLUX_INC_NAME # ) # print(down_flux_inc_matrix_w_m03[0, ...]) # print('\n') prediction_example_dict = normalization.denormalize_data( new_example_dict=prediction_example_dict, training_example_dict=training_example_dict, normalization_type_string=vector_target_norm_type_string, min_normalized_value=generator_option_dict[ neural_net.VECTOR_TARGET_MIN_VALUE_KEY], max_normalized_value=generator_option_dict[ neural_net.VECTOR_TARGET_MAX_VALUE_KEY], separate_heights=True, apply_to_predictors=False, apply_to_vector_targets=True, apply_to_scalar_targets=False) # down_flux_inc_matrix_w_m03 = example_utils.get_field_from_dict( # example_dict=prediction_example_dict, # field_name=example_utils.SHORTWAVE_DOWN_FLUX_INC_NAME # ) # print(down_flux_inc_matrix_w_m03[0, ...]) # print('\n\n\n') if scalar_target_norm_type_string is not None: print('Denormalizing predicted scalars...') prediction_example_dict = normalization.denormalize_data( new_example_dict=prediction_example_dict, training_example_dict=training_example_dict, normalization_type_string=scalar_target_norm_type_string, min_normalized_value=generator_option_dict[ neural_net.SCALAR_TARGET_MIN_VALUE_KEY], max_normalized_value=generator_option_dict[ neural_net.SCALAR_TARGET_MAX_VALUE_KEY], separate_heights=True, apply_to_predictors=False, apply_to_vector_targets=False, apply_to_scalar_targets=True) add_heating_rate = generator_option_dict[neural_net.OMIT_HEATING_RATE_KEY] if add_heating_rate: pressure_matrix_pascals = _get_unnormalized_pressure( model_metadata_dict=metadata_dict, example_id_strings=example_id_strings) prediction_example_dict = _get_predicted_heating_rates( prediction_example_dict=prediction_example_dict, pressure_matrix_pascals=pressure_matrix_pascals, model_metadata_dict=metadata_dict) vector_target_names = ( generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY]) if example_utils.SHORTWAVE_HEATING_RATE_NAME in vector_target_names: heating_rate_index = vector_target_names.index( example_utils.SHORTWAVE_HEATING_RATE_NAME) heights_m_agl = generator_option_dict[neural_net.HEIGHTS_KEY] height_indices = numpy.where( heights_m_agl >= ZERO_HEATING_HEIGHT_M_AGL)[0] vector_target_matrix = ( prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY]) vector_target_matrix[..., heating_rate_index][..., height_indices] = 0. prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY] = ( vector_target_matrix) all_heights_m_agl = generator_option_dict[neural_net.HEIGHTS_KEY] desired_heights_m_agl = ( all_heights_m_agl[all_heights_m_agl < MAX_HEIGHT_M_AGL]) target_example_dict = example_utils.subset_by_height( example_dict=target_example_dict, heights_m_agl=desired_heights_m_agl) prediction_example_dict = example_utils.subset_by_height( example_dict=prediction_example_dict, heights_m_agl=desired_heights_m_agl) print('Writing target (actual) and predicted values to: "{0:s}"...'.format( output_file_name)) prediction_io.write_file(netcdf_file_name=output_file_name, scalar_target_matrix=target_example_dict[ example_utils.SCALAR_TARGET_VALS_KEY], vector_target_matrix=target_example_dict[ example_utils.VECTOR_TARGET_VALS_KEY], scalar_prediction_matrix=prediction_example_dict[ example_utils.SCALAR_TARGET_VALS_KEY], vector_prediction_matrix=prediction_example_dict[ example_utils.VECTOR_TARGET_VALS_KEY], heights_m_agl=desired_heights_m_agl, example_id_strings=example_id_strings, model_file_name=model_file_name)
def read_file(netcdf_file_name): """Reads predictions from NetCDF file. :param netcdf_file_name: Path to input file. :return: prediction_dict: Dictionary with the following keys. prediction_dict['scalar_target_matrix']: See doc for `write_file`. prediction_dict['scalar_prediction_matrix']: Same. prediction_dict['vector_target_matrix']: Same. prediction_dict['vector_prediction_matrix']: Same. prediction_dict['example_id_strings']: Same. prediction_dict['model_file_name']: Same. prediction_dict['isotonic_model_file_name']: Same. """ dataset_object = netCDF4.Dataset(netcdf_file_name) prediction_dict = { VECTOR_TARGETS_KEY: dataset_object.variables[VECTOR_TARGETS_KEY][:], VECTOR_PREDICTIONS_KEY: dataset_object.variables[VECTOR_PREDICTIONS_KEY][:], EXAMPLE_IDS_KEY: [ str(id) for id in netCDF4.chartostring( dataset_object.variables[EXAMPLE_IDS_KEY][:]) ], MODEL_FILE_KEY: str(getattr(dataset_object, MODEL_FILE_KEY)) } if ISOTONIC_MODEL_FILE_KEY in prediction_dict: prediction_dict[ISOTONIC_MODEL_FILE_KEY] = str( getattr(dataset_object, ISOTONIC_MODEL_FILE_KEY)) else: prediction_dict[ISOTONIC_MODEL_FILE_KEY] = '' if prediction_dict[ISOTONIC_MODEL_FILE_KEY] == '': prediction_dict[ISOTONIC_MODEL_FILE_KEY] = None if HEIGHTS_KEY in dataset_object.variables: prediction_dict[HEIGHTS_KEY] = dataset_object.variables[HEIGHTS_KEY][:] else: model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(prediction_dict[MODEL_FILE_KEY])[0], raise_error_if_missing=True) model_metadata_dict = neural_net.read_metafile(model_metafile_name) generator_option_dict = ( model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY]) prediction_dict[HEIGHTS_KEY] = ( generator_option_dict[neural_net.HEIGHTS_KEY]) if SCALAR_TARGETS_KEY in dataset_object.variables: prediction_dict[SCALAR_TARGETS_KEY] = ( dataset_object.variables[SCALAR_TARGETS_KEY][:]) prediction_dict[SCALAR_PREDICTIONS_KEY] = ( dataset_object.variables[SCALAR_PREDICTIONS_KEY][:]) else: num_examples = prediction_dict[VECTOR_TARGETS_KEY].shape[0] prediction_dict[SCALAR_TARGETS_KEY] = numpy.full((num_examples, 0), 0.) prediction_dict[SCALAR_PREDICTIONS_KEY] = numpy.full((num_examples, 0), 0.) dataset_object.close() return prediction_dict
def _run(prediction_file_name, num_examples, example_dir_name, use_log_scale, output_dir_name): """Plots comparisons between predicted and actual (target) profiles. This is effectively the main method. :param prediction_file_name: See documentation at top of file. :param num_examples: Same. :param example_dir_name: Same. :param use_log_scale: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) if num_examples < 1: num_examples = None if example_dir_name == '': example_dir_name = None print(('Reading predicted and actual (target) profiles from: "{0:s}"...' ).format(prediction_file_name)) prediction_dict = prediction_io.read_file(prediction_file_name) num_examples_orig = len(prediction_dict[prediction_io.EXAMPLE_IDS_KEY]) if num_examples is not None and num_examples < num_examples_orig: desired_indices = numpy.linspace(0, num_examples - 1, num=num_examples, dtype=int) prediction_dict = prediction_io.subset_by_index( prediction_dict=prediction_dict, desired_indices=desired_indices) vector_target_matrix = prediction_dict[prediction_io.VECTOR_TARGETS_KEY] vector_prediction_matrix = ( prediction_dict[prediction_io.VECTOR_PREDICTIONS_KEY]) scalar_target_matrix = prediction_dict[prediction_io.SCALAR_TARGETS_KEY] scalar_prediction_matrix = ( prediction_dict[prediction_io.SCALAR_PREDICTIONS_KEY]) model_file_name = prediction_dict[prediction_io.MODEL_FILE_KEY] model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY][ neural_net.HEIGHTS_KEY] = prediction_dict[prediction_io.HEIGHTS_KEY] # If necessary, convert flux increments to fluxes. vector_target_matrix, vector_prediction_matrix, model_metadata_dict = ( _fluxes_increments_to_actual( vector_target_matrix=vector_target_matrix, vector_prediction_matrix=vector_prediction_matrix, model_metadata_dict=model_metadata_dict)) # If necessary, convert fluxes to heating rates. vector_target_matrix, vector_prediction_matrix, model_metadata_dict = ( _fluxes_to_heating_rate( vector_target_matrix=vector_target_matrix, vector_prediction_matrix=vector_prediction_matrix, model_metadata_dict=model_metadata_dict, prediction_file_name=prediction_file_name, example_dir_name=example_dir_name)) # If data include both upwelling and downwelling fluxes, remove flux # increments (they need not be plotted). vector_target_matrix, vector_prediction_matrix, model_metadata_dict = ( _remove_flux_increments( vector_target_matrix=vector_target_matrix, vector_prediction_matrix=vector_prediction_matrix, model_metadata_dict=model_metadata_dict)) generator_option_dict = model_metadata_dict[ neural_net.TRAINING_OPTIONS_KEY] vector_target_names = ( generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY]) plot_fancy = all( [t in vector_target_names for t in DEFAULT_VECTOR_TARGET_NAMES]) if plot_fancy: _plot_comparisons_fancy( vector_target_matrix=vector_target_matrix, vector_prediction_matrix=vector_prediction_matrix, example_id_strings=prediction_dict[prediction_io.EXAMPLE_IDS_KEY], model_metadata_dict=model_metadata_dict, use_log_scale=use_log_scale, output_dir_name=output_dir_name) else: title_strings = _get_flux_strings( scalar_target_matrix=scalar_target_matrix, scalar_prediction_matrix=scalar_prediction_matrix, model_metadata_dict=model_metadata_dict) _plot_comparisons_simple( vector_target_matrix=vector_target_matrix, vector_prediction_matrix=vector_prediction_matrix, example_id_strings=prediction_dict[prediction_io.EXAMPLE_IDS_KEY], model_metadata_dict=model_metadata_dict, use_log_scale=use_log_scale, title_strings=title_strings, output_dir_name=output_dir_name)
def _run(model_file_name, example_file_name, num_examples, example_dir_name, example_id_file_name, activation_layer_name, vector_output_layer_name, output_neuron_indices, ideal_activation, output_file_name): """Runs the Grad-CAM (gradient-weighted class-activation maps) algorithm. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_file_name: Same. :param num_examples: Same. :param example_dir_name: Same. :param example_id_file_name: Same. :param activation_layer_name: Same. :param vector_output_layer_name: Same. :param output_neuron_indices: Same. :param ideal_activation: Same. :param output_file_name: Same. :raises: ValueError: if neural-net type is not CNN or U-net. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) predictor_matrix, _, example_id_strings = ( misc_utils.get_examples_for_inference( model_metadata_dict=metadata_dict, example_file_name=example_file_name, num_examples=num_examples, example_dir_name=example_dir_name, example_id_file_name=example_id_file_name)) print(SEPARATOR_STRING) net_type_string = metadata_dict[neural_net.NET_TYPE_KEY] valid_net_type_strings = [ neural_net.CNN_TYPE_STRING, neural_net.U_NET_TYPE_STRING ] if net_type_string not in valid_net_type_strings: error_string = ( '\nThis script does not work for net type "{0:s}". Works only for ' 'those listed below:\n{1:s}').format(net_type_string, str(valid_net_type_strings)) raise ValueError(error_string) num_examples = predictor_matrix.shape[0] num_heights = predictor_matrix.shape[1] class_activation_matrix = numpy.full((num_examples, num_heights), numpy.nan) for i in range(num_examples): if numpy.mod(i, 10) == 0: print('Have run Grad-CAM for {0:d} of {1:d} examples...'.format( i, num_examples)) class_activation_matrix[i, :] = gradcam.run_gradcam( model_object=model_object, predictor_matrix=predictor_matrix[i, ...], activation_layer_name=activation_layer_name, vector_output_layer_name=vector_output_layer_name, output_neuron_indices=output_neuron_indices, ideal_activation=ideal_activation) print('Have run Grad-CAM for all {0:d} examples!\n'.format(num_examples)) print('Writing class-activation maps to: "{0:s}"...'.format( output_file_name)) gradcam.write_file(netcdf_file_name=output_file_name, class_activation_matrix=class_activation_matrix, example_id_strings=example_id_strings, model_file_name=model_file_name, activation_layer_name=activation_layer_name, vector_output_layer_name=vector_output_layer_name, output_neuron_indices=output_neuron_indices, ideal_activation=ideal_activation)
def _run(input_prediction_file_name, average_over_height, scale_by_climo, num_examples_per_set, output_dir_name): """Finds best and worst heating-rate predictions. This is effectively the main method. :param input_prediction_file_name: See documentation at top of file. :param average_over_height: Same. :param scale_by_climo: Same. :param num_examples_per_set: Same. :param output_dir_name: Same. """ # TODO(thunderhoser): Maybe allow specific height again (e.g., 15 km). error_checking.assert_is_greater(num_examples_per_set, 0) scale_by_climo = scale_by_climo and not average_over_height print('Reading data from: "{0:s}"...'.format(input_prediction_file_name)) prediction_dict = prediction_io.read_file(input_prediction_file_name) model_file_name = prediction_dict[prediction_io.MODEL_FILE_KEY] model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0]) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) generator_option_dict = model_metadata_dict[ neural_net.TRAINING_OPTIONS_KEY] vector_target_names = ( generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY]) hr_index = (vector_target_names.index( example_utils.SHORTWAVE_HEATING_RATE_NAME)) target_matrix_k_day01 = ( prediction_dict[prediction_io.VECTOR_TARGETS_KEY][..., hr_index]) prediction_matrix_k_day01 = ( prediction_dict[prediction_io.VECTOR_PREDICTIONS_KEY][..., hr_index]) bias_matrix = prediction_matrix_k_day01 - target_matrix_k_day01 absolute_error_matrix = numpy.absolute(bias_matrix) if average_over_height: bias_matrix = numpy.mean(bias_matrix, axis=1, keepdims=True) absolute_error_matrix = numpy.mean(absolute_error_matrix, axis=1, keepdims=True) if scale_by_climo: normalization_file_name = ( generator_option_dict[neural_net.NORMALIZATION_FILE_KEY]) print(('Reading training examples (for climatology) from: "{0:s}"...' ).format(normalization_file_name)) training_example_dict = example_io.read_file(normalization_file_name) training_example_dict = example_utils.subset_by_field( example_dict=training_example_dict, field_names=[example_utils.SHORTWAVE_HEATING_RATE_NAME]) training_example_dict = example_utils.subset_by_height( example_dict=training_example_dict, heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY]) dummy_example_dict = { example_utils.SCALAR_PREDICTOR_NAMES_KEY: [], example_utils.VECTOR_PREDICTOR_NAMES_KEY: [], example_utils.SCALAR_TARGET_NAMES_KEY: [], example_utils.VECTOR_TARGET_NAMES_KEY: [example_utils.SHORTWAVE_HEATING_RATE_NAME], example_utils.HEIGHTS_KEY: generator_option_dict[neural_net.HEIGHTS_KEY] } mean_training_example_dict = normalization.create_mean_example( new_example_dict=dummy_example_dict, training_example_dict=training_example_dict) climo_matrix_k_day01 = mean_training_example_dict[ example_utils.VECTOR_TARGET_VALS_KEY][..., 0] bias_matrix = bias_matrix / climo_matrix_k_day01 absolute_error_matrix = absolute_error_matrix / climo_matrix_k_day01 print(SEPARATOR_STRING) high_bias_indices, low_bias_indices, low_abs_error_indices = ( misc_utils.find_best_and_worst_predictions( bias_matrix=bias_matrix, absolute_error_matrix=absolute_error_matrix, num_examples_per_set=num_examples_per_set)) print(SEPARATOR_STRING) high_bias_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=high_bias_indices) high_bias_file_name = ( '{0:s}/predictions_high-bias.nc'.format(output_dir_name)) print('Writing examples with greatest positive bias to: "{0:s}"...'.format( high_bias_file_name)) prediction_io.write_file( netcdf_file_name=high_bias_file_name, scalar_target_matrix=high_bias_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=high_bias_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=high_bias_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=high_bias_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=high_bias_prediction_dict[prediction_io.HEIGHTS_KEY], example_id_strings=high_bias_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=high_bias_prediction_dict[ prediction_io.MODEL_FILE_KEY]) low_bias_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=low_bias_indices) low_bias_file_name = ( '{0:s}/predictions_low-bias.nc'.format(output_dir_name)) print('Writing examples with greatest negative bias to: "{0:s}"...'.format( low_bias_file_name)) prediction_io.write_file( netcdf_file_name=low_bias_file_name, scalar_target_matrix=low_bias_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=low_bias_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=low_bias_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=low_bias_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=low_bias_prediction_dict[prediction_io.HEIGHTS_KEY], example_id_strings=low_bias_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=low_bias_prediction_dict[prediction_io.MODEL_FILE_KEY]) low_abs_error_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=low_abs_error_indices) low_abs_error_file_name = ( '{0:s}/predictions_low-absolute-error.nc'.format(output_dir_name)) print( 'Writing examples with smallest absolute error to: "{0:s}"...'.format( low_abs_error_file_name)) prediction_io.write_file( netcdf_file_name=low_abs_error_file_name, scalar_target_matrix=low_abs_error_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=low_abs_error_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=low_abs_error_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=low_abs_error_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=low_abs_error_prediction_dict[prediction_io.HEIGHTS_KEY], example_id_strings=low_abs_error_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=low_abs_error_prediction_dict[ prediction_io.MODEL_FILE_KEY]) if scale_by_climo: return if average_over_height: mean_targets_k_day01 = numpy.mean(target_matrix_k_day01, axis=1) sort_indices = numpy.argsort(-1 * mean_targets_k_day01) else: max_targets_k_day01 = numpy.max(target_matrix_k_day01, axis=1) sort_indices = numpy.argsort(-1 * max_targets_k_day01) large_hr_indices = sort_indices[:num_examples_per_set] large_hr_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=large_hr_indices) large_hr_file_name = ( '{0:s}/predictions_large-heating-rate.nc'.format(output_dir_name)) print('Writing examples with greatest heating rate to: "{0:s}"...'.format( large_hr_file_name)) prediction_io.write_file( netcdf_file_name=large_hr_file_name, scalar_target_matrix=large_hr_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=large_hr_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=large_hr_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=large_hr_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=large_hr_prediction_dict[prediction_io.HEIGHTS_KEY], example_id_strings=large_hr_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=large_hr_prediction_dict[prediction_io.MODEL_FILE_KEY]) if not average_over_height: return mean_targets_k_day01 = numpy.mean(target_matrix_k_day01, axis=1) sort_indices = numpy.argsort(mean_targets_k_day01) small_hr_indices = sort_indices[:num_examples_per_set] small_hr_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=small_hr_indices) small_hr_file_name = ( '{0:s}/predictions_small-heating-rate.nc'.format(output_dir_name)) print('Writing examples with smallest heating rate to: "{0:s}"...'.format( small_hr_file_name)) prediction_io.write_file( netcdf_file_name=small_hr_file_name, scalar_target_matrix=small_hr_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=small_hr_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=small_hr_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=small_hr_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=small_hr_prediction_dict[prediction_io.HEIGHTS_KEY], example_id_strings=small_hr_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=small_hr_prediction_dict[prediction_io.MODEL_FILE_KEY])
def _run(model_file_name, example_file_name, num_examples, example_dir_name, example_id_file_name, activation_layer_name, vector_output_layer_name, ideal_activation, output_file_name): """Runs Grad-CAM for each example and target variable. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_file_name: Same. :param num_examples: Same. :param example_dir_name: Same. :param example_id_file_name: Same. :param activation_layer_name: Same. :param vector_output_layer_name: Same. :param ideal_activation: Same. :param output_file_name: Same. :raises: ValueError: if neural-net type is not CNN or U-net. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True ) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) generator_option_dict = metadata_dict[neural_net.TRAINING_OPTIONS_KEY] net_type_string = metadata_dict[neural_net.NET_TYPE_KEY] valid_net_type_strings = [ neural_net.CNN_TYPE_STRING, neural_net.U_NET_TYPE_STRING ] if net_type_string not in valid_net_type_strings: error_string = ( '\nThis script does not work for net type "{0:s}". Works only for ' 'those listed below:\n{1:s}' ).format(net_type_string, str(valid_net_type_strings)) raise ValueError(error_string) predictor_matrix, _, example_id_strings = ( misc_utils.get_examples_for_inference( model_metadata_dict=metadata_dict, example_file_name=example_file_name, num_examples=num_examples, example_dir_name=example_dir_name, example_id_file_name=example_id_file_name ) ) print(SEPARATOR_STRING) vector_target_names = ( generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY] ) heights_m_agl = generator_option_dict[neural_net.HEIGHTS_KEY] dummy_example_dict = { example_utils.SCALAR_TARGET_NAMES_KEY: [], example_utils.VECTOR_TARGET_NAMES_KEY: vector_target_names, example_utils.HEIGHTS_KEY: heights_m_agl } num_examples = len(example_id_strings) num_vector_targets = len(vector_target_names) num_heights = len(heights_m_agl) class_activation_matrix = numpy.full( (num_examples, num_heights, num_heights, num_vector_targets), numpy.nan ) for i in range(num_examples): print(( 'Have run Grad-CAM (all target variables) for {0:d} of {1:d} ' 'examples...' ).format( i, num_examples )) for k in range(num_vector_targets): for j in range(num_heights): these_neuron_indices = neural_net.target_var_to_neuron_indices( example_dict=dummy_example_dict, net_type_string=net_type_string, target_name=vector_target_names[k], height_m_agl=heights_m_agl[j] ) class_activation_matrix[i, :, j, k] = gradcam.run_gradcam( model_object=model_object, predictor_matrix=predictor_matrix[i, ...], activation_layer_name=activation_layer_name, vector_output_layer_name=vector_output_layer_name, output_neuron_indices=these_neuron_indices, ideal_activation=ideal_activation ) print(( 'Have run Grad-CAM (all target variables) for all {0:d} examples!\n' ).format( num_examples )) print('Writing class-activation maps to: "{0:s}"...'.format( output_file_name )) gradcam.write_all_targets_file( netcdf_file_name=output_file_name, class_activation_matrix=class_activation_matrix, example_id_strings=example_id_strings, model_file_name=model_file_name, activation_layer_name=activation_layer_name, vector_output_layer_name=vector_output_layer_name, ideal_activation=ideal_activation )
def _run(example_file_name, num_examples, example_dir_name, example_id_file_name, use_log_scale, model_file_name, output_dir_name): """Plots profiles (vector predictor and target variables) for each example. This is effectively the main method. :param example_file_name: See documentation at top of file. :param num_examples: Same. :param example_dir_name: Same. :param example_id_file_name: Same. :param use_log_scale: Same. :param model_file_name: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) example_dict = misc_utils.get_raw_examples( example_file_name=example_file_name, num_examples=num_examples, example_dir_name=example_dir_name, example_id_file_name=example_id_file_name) num_examples_total = len(example_dict[example_utils.VALID_TIMES_KEY]) if 0 < num_examples < num_examples_total: desired_indices = numpy.linspace(0, num_examples - 1, num=num_examples, dtype=int) example_dict = example_utils.subset_by_index( example_dict=example_dict, desired_indices=desired_indices) if model_file_name != '': model_metafile_name = neural_net.find_metafile( os.path.split(model_file_name)[0]) print('Reading model metadata from: "{0:s}"...\n'.format( model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) generator_option_dict = ( model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY]) vector_predictor_names = ( generator_option_dict[neural_net.VECTOR_PREDICTOR_NAMES_KEY]) all_field_names = vector_predictor_names + VECTOR_TARGET_NAMES example_dict = example_utils.subset_by_field( example_dict=example_dict, field_names=all_field_names) example_dict = example_utils.subset_by_height( example_dict=example_dict, heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY]) num_examples = len(example_dict[example_utils.VALID_TIMES_KEY]) for i in range(num_examples): _plot_one_example(example_dict=example_dict, example_index=i, use_log_scale=use_log_scale, output_dir_name=output_dir_name) print(MINOR_SEPARATOR_STRING)
def _run(input_prediction_file_name, num_examples_per_set, output_dir_name): """Finds best and worst heating-rate predictions. This is effectively the main method. :param input_prediction_file_name: See documentation at top of file. :param num_examples_per_set: Same. :param output_dir_name: Same. """ error_checking.assert_is_greater(num_examples_per_set, 0) print('Reading data from: "{0:s}"...'.format(input_prediction_file_name)) prediction_dict = prediction_io.read_file(input_prediction_file_name) model_file_name = prediction_dict[prediction_io.MODEL_FILE_KEY] model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0]) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) generator_option_dict = model_metadata_dict[ neural_net.TRAINING_OPTIONS_KEY] scalar_target_names = ( generator_option_dict[neural_net.SCALAR_TARGET_NAMES_KEY]) down_index = scalar_target_names.index( example_utils.SHORTWAVE_SURFACE_DOWN_FLUX_NAME) up_index = scalar_target_names.index( example_utils.SHORTWAVE_TOA_UP_FLUX_NAME) targets_w_m02 = ( prediction_dict[prediction_io.SCALAR_TARGETS_KEY][..., down_index] - prediction_dict[prediction_io.SCALAR_TARGETS_KEY][..., up_index]) predictions_w_m02 = ( prediction_dict[prediction_io.SCALAR_PREDICTIONS_KEY][..., down_index] - prediction_dict[prediction_io.SCALAR_PREDICTIONS_KEY][..., up_index]) biases_w_m02 = predictions_w_m02 - targets_w_m02 bias_matrix = numpy.expand_dims(biases_w_m02, axis=1) print(SEPARATOR_STRING) high_bias_indices, low_bias_indices, low_abs_error_indices = ( misc_utils.find_best_and_worst_predictions( bias_matrix=bias_matrix, absolute_error_matrix=numpy.absolute(bias_matrix), num_examples_per_set=num_examples_per_set)) print(SEPARATOR_STRING) high_bias_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=high_bias_indices) high_bias_file_name = ( '{0:s}/predictions_high-bias.nc'.format(output_dir_name)) print('Writing examples with greatest positive bias to: "{0:s}"...'.format( high_bias_file_name)) prediction_io.write_file( netcdf_file_name=high_bias_file_name, scalar_target_matrix=high_bias_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=high_bias_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=high_bias_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=high_bias_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=high_bias_prediction_dict[prediction_io.HEIGHTS_KEY], example_id_strings=high_bias_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=high_bias_prediction_dict[ prediction_io.MODEL_FILE_KEY]) low_bias_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=low_bias_indices) low_bias_file_name = ( '{0:s}/predictions_low-bias.nc'.format(output_dir_name)) print('Writing examples with greatest negative bias to: "{0:s}"...'.format( low_bias_file_name)) prediction_io.write_file( netcdf_file_name=low_bias_file_name, scalar_target_matrix=low_bias_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=low_bias_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=low_bias_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=low_bias_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=low_bias_prediction_dict[prediction_io.HEIGHTS_KEY], example_id_strings=low_bias_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=low_bias_prediction_dict[prediction_io.MODEL_FILE_KEY]) low_abs_error_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=low_abs_error_indices) low_abs_error_file_name = ( '{0:s}/predictions_low-absolute-error.nc'.format(output_dir_name)) print( 'Writing examples with smallest absolute error to: "{0:s}"...'.format( low_abs_error_file_name)) prediction_io.write_file( netcdf_file_name=low_abs_error_file_name, scalar_target_matrix=low_abs_error_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=low_abs_error_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=low_abs_error_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=low_abs_error_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=low_abs_error_prediction_dict[prediction_io.HEIGHTS_KEY], example_id_strings=low_abs_error_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=low_abs_error_prediction_dict[ prediction_io.MODEL_FILE_KEY]) sort_indices = numpy.argsort(-1 * targets_w_m02) large_net_flux_indices = sort_indices[:num_examples_per_set] large_net_flux_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=large_net_flux_indices) large_net_flux_file_name = ( '{0:s}/predictions_large-net-flux.nc'.format(output_dir_name)) print('Writing examples with greatest net flux to: "{0:s}"...'.format( large_net_flux_file_name)) prediction_io.write_file( netcdf_file_name=large_net_flux_file_name, scalar_target_matrix=large_net_flux_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=large_net_flux_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=large_net_flux_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=large_net_flux_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=large_net_flux_prediction_dict[ prediction_io.HEIGHTS_KEY], example_id_strings=large_net_flux_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=large_net_flux_prediction_dict[ prediction_io.MODEL_FILE_KEY]) sort_indices = numpy.argsort(targets_w_m02) small_net_flux_indices = sort_indices[:num_examples_per_set] small_net_flux_prediction_dict = prediction_io.subset_by_index( prediction_dict=copy.deepcopy(prediction_dict), desired_indices=small_net_flux_indices) small_net_flux_file_name = ( '{0:s}/predictions_small-net-flux.nc'.format(output_dir_name)) print('Writing examples with smallest net flux to: "{0:s}"...'.format( small_net_flux_file_name)) prediction_io.write_file( netcdf_file_name=small_net_flux_file_name, scalar_target_matrix=small_net_flux_prediction_dict[ prediction_io.SCALAR_TARGETS_KEY], vector_target_matrix=small_net_flux_prediction_dict[ prediction_io.VECTOR_TARGETS_KEY], scalar_prediction_matrix=small_net_flux_prediction_dict[ prediction_io.SCALAR_PREDICTIONS_KEY], vector_prediction_matrix=small_net_flux_prediction_dict[ prediction_io.VECTOR_PREDICTIONS_KEY], heights_m_agl=small_net_flux_prediction_dict[ prediction_io.HEIGHTS_KEY], example_id_strings=small_net_flux_prediction_dict[ prediction_io.EXAMPLE_IDS_KEY], model_file_name=small_net_flux_prediction_dict[ prediction_io.MODEL_FILE_KEY])
def _run(gradcam_file_name, use_log_scale, prediction_file_name, output_dir_name): """Plots Grad-CAM output (class-activation maps). This is effectively the main method. :param gradcam_file_name: See documentation at top of file. :param use_log_scale: Same. :param prediction_file_name: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name ) print('Reading class-activation maps from: "{0:s}"...'.format( gradcam_file_name )) gradcam_dict = gradcam.read_file(gradcam_file_name) example_id_strings = gradcam_dict[gradcam.EXAMPLE_IDS_KEY] output_neuron_indices = gradcam_dict[gradcam.OUTPUT_NEURONS_KEY] model_file_name = gradcam_dict[gradcam.MODEL_FILE_KEY] model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0] ) print('Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) generator_option_dict = model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY] dummy_example_dict = { example_utils.VECTOR_TARGET_NAMES_KEY: generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY], example_utils.HEIGHTS_KEY: generator_option_dict[neural_net.HEIGHTS_KEY] } target_field_name, target_height_m_agl = ( neural_net.neuron_indices_to_target_var( neuron_indices=output_neuron_indices, example_dict=dummy_example_dict, net_type_string=model_metadata_dict[neural_net.NET_TYPE_KEY] ) ) predicted_target_values, actual_target_values = ( plot_saliency_maps._get_target_values( prediction_file_name=prediction_file_name, model_metadata_dict=model_metadata_dict, example_id_strings=example_id_strings, target_field_name=target_field_name, target_height_m_agl=target_height_m_agl ) ) print(SEPARATOR_STRING) num_examples = len(example_id_strings) for i in range(num_examples): this_title_string = 'Actual and predicted {0:s}'.format( target_field_name ) if target_height_m_agl is not None: this_title_string += ' at {0:.2f} km AGL'.format( METRES_TO_KM * target_height_m_agl ) this_title_string += ' = {0:.2f}, {1:.2f}'.format( actual_target_values[i], predicted_target_values[i] ) _plot_one_example( gradcam_dict=gradcam_dict, example_index=i, model_metadata_dict=model_metadata_dict, use_log_scale=use_log_scale, title_string=this_title_string, output_dir_name=output_dir_name )
def _run(saliency_file_name, use_log_scale, prediction_file_name, output_dir_name): """Plots saliency maps (one for each example). This is effectively the main method. :param saliency_file_name: See documentation at top of file. :param use_log_scale: Same. :param prediction_file_name: Same. :param output_dir_name: Same. """ file_system_utils.mkdir_recursive_if_necessary( directory_name=output_dir_name) print( 'Reading saliency values from: "{0:s}"...'.format(saliency_file_name)) saliency_dict = saliency.read_file(saliency_file_name) target_field_name = saliency_dict[saliency.TARGET_FIELD_KEY] target_height_m_agl = saliency_dict[saliency.TARGET_HEIGHT_KEY] example_id_strings = saliency_dict[saliency.EXAMPLE_IDS_KEY] model_file_name = saliency_dict[saliency.MODEL_FILE_KEY] model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0]) print( 'Reading model metadata from: "{0:s}"...'.format(model_metafile_name)) model_metadata_dict = neural_net.read_metafile(model_metafile_name) num_examples = len(example_id_strings) if target_field_name is None or prediction_file_name == '': predicted_target_values = [None] * num_examples actual_target_values = [None] * num_examples else: predicted_target_values, actual_target_values = _get_target_values( prediction_file_name=prediction_file_name, model_metadata_dict=model_metadata_dict, example_id_strings=example_id_strings, target_field_name=target_field_name, target_height_m_agl=target_height_m_agl) print(SEPARATOR_STRING) for i in range(num_examples): if target_field_name is None: this_legend_suffix = '' else: this_height_string = ('' if target_height_m_agl is None else '{0:.2f}-km '.format(METRES_TO_KM * target_height_m_agl)) this_legend_suffix = ( 'A&P {0:s}{1:s} = {2:.2f}, {3:.2f} {4:s}').format( this_height_string, TARGET_NAME_TO_VERBOSE[target_field_name], actual_target_values[i], predicted_target_values[i], TARGET_NAME_TO_UNITS[target_field_name]) _plot_saliency_one_example(saliency_dict=saliency_dict, example_index=i, model_metadata_dict=model_metadata_dict, use_log_scale=use_log_scale, legend_suffix=this_legend_suffix, output_dir_name=output_dir_name) if i != num_examples - 1: print('\n')
def _run(model_file_name, example_file_name, num_examples, example_dir_name, example_id_file_name, heating_rate_weight, flux_weight, include_net_flux, do_backwards_test, shuffle_profiles_together, num_bootstrap_reps, output_file_name): """Runs permutation-based importance test. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_file_name: Same. :param num_examples: Same. :param example_dir_name: Same. :param example_id_file_name: Same. :param heating_rate_weight: Same. :param flux_weight: Same. :param include_net_flux: Same. :param do_backwards_test: Same. :param shuffle_profiles_together: Same. :param num_bootstrap_reps: Same. :param output_file_name: Same. """ cost_function = permutation.make_cost_function( heating_rate_weight=heating_rate_weight, flux_weight=flux_weight, include_net_flux=include_net_flux ) print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True ) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) predictor_matrix, target_matrices = ( misc_utils.get_examples_for_inference( model_metadata_dict=metadata_dict, example_file_name=example_file_name, num_examples=num_examples, example_dir_name=example_dir_name, example_id_file_name=example_id_file_name )[:2] ) print(SEPARATOR_STRING) if not isinstance(target_matrices, list): target_matrices = [target_matrices] if do_backwards_test: result_dict = permutation.run_backwards_test( predictor_matrix=predictor_matrix, target_matrices=target_matrices, model_object=model_object, model_metadata_dict=metadata_dict, cost_function=cost_function, shuffle_profiles_together=shuffle_profiles_together, num_bootstrap_reps=num_bootstrap_reps ) else: result_dict = permutation.run_forward_test( predictor_matrix=predictor_matrix, target_matrices=target_matrices, model_object=model_object, model_metadata_dict=metadata_dict, cost_function=cost_function, shuffle_profiles_together=shuffle_profiles_together, num_bootstrap_reps=num_bootstrap_reps ) print(SEPARATOR_STRING) print('Writing results of permutation test to: "{0:s}"...'.format( output_file_name )) permutation.write_file( result_dict=result_dict, netcdf_file_name=output_file_name )
def _run(model_file_name, example_file_name, num_examples, example_dir_name, example_id_file_name, layer_name, neuron_indices, ideal_activation, num_iterations, learning_rate, l2_weight, output_file_name): """Runs backwards optimization. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_file_name: Same. :param num_examples: Same. :param example_dir_name: Same. :param example_id_file_name: Same. :param layer_name: Same. :param neuron_indices: Same. :param ideal_activation: Same. :param num_iterations: Same. :param learning_rate: Same. :param l2_weight: Same. :param output_file_name: Same. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True ) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) predictor_matrix, _, example_id_strings = ( misc_utils.get_examples_for_inference( model_metadata_dict=metadata_dict, example_file_name=example_file_name, num_examples=num_examples, example_dir_name=example_dir_name, example_id_file_name=example_id_file_name ) ) print(SEPARATOR_STRING) generator_option_dict = metadata_dict[neural_net.TRAINING_OPTIONS_KEY] normalization_file_name = ( generator_option_dict[neural_net.NORMALIZATION_FILE_KEY] ) print(( 'Reading training examples (for normalization) from: "{0:s}"...' ).format( normalization_file_name )) training_example_dict = example_io.read_file(normalization_file_name) training_example_dict = example_utils.subset_by_height( example_dict=training_example_dict, heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY] ) num_examples = len(example_id_strings) bwo_dict = None for i in range(num_examples): this_bwo_dict = bwo.optimize_input_for_neuron( model_object=model_object, init_function_or_matrix=predictor_matrix[i, ...], layer_name=layer_name, neuron_indices=neuron_indices, ideal_activation=ideal_activation, num_iterations=num_iterations, learning_rate=learning_rate, l2_weight=l2_weight ) if i == num_examples - 1: print(SEPARATOR_STRING) else: print(MINOR_SEPARATOR_STRING) if bwo_dict is None: these_dim = numpy.array( (num_examples,) + this_bwo_dict[bwo.INITIAL_PREDICTORS_KEY].shape[1:], dtype=int ) bwo_dict = { bwo.INITIAL_PREDICTORS_KEY: numpy.full(these_dim, numpy.nan), bwo.FINAL_PREDICTORS_KEY: numpy.full(these_dim, numpy.nan), bwo.INITIAL_ACTIVATIONS_KEY: numpy.full(num_examples, numpy.nan), bwo.FINAL_ACTIVATIONS_KEY: numpy.full(num_examples, numpy.nan) } bwo_dict[bwo.INITIAL_PREDICTORS_KEY][i, ...] = ( this_bwo_dict[bwo.INITIAL_PREDICTORS_KEY][0, ...] ) bwo_dict[bwo.FINAL_PREDICTORS_KEY][i, ...] = ( this_bwo_dict[bwo.FINAL_PREDICTORS_KEY][0, ...] ) bwo_dict[bwo.INITIAL_ACTIVATIONS_KEY][i] = ( this_bwo_dict[bwo.INITIAL_ACTIVATION_KEY] ) bwo_dict[bwo.FINAL_ACTIVATIONS_KEY][i] = ( this_bwo_dict[bwo.FINAL_ACTIVATION_KEY] ) if example_file_name == '': example_file_name = example_io.find_many_files( directory_name=example_dir_name, first_time_unix_sec=0, last_time_unix_sec=int(1e12), raise_error_if_any_missing=False, raise_error_if_all_missing=True )[0] first_example_dict = example_io.read_file(example_file_name) first_example_dict = example_utils.subset_by_height( example_dict=first_example_dict, heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY] ) net_type_string = metadata_dict[neural_net.NET_TYPE_KEY] init_example_dict = copy.deepcopy(first_example_dict) this_example_dict = neural_net.predictors_numpy_to_dict( predictor_matrix=bwo_dict[bwo.INITIAL_PREDICTORS_KEY], example_dict=init_example_dict, net_type_string=net_type_string ) init_example_dict.update(this_example_dict) if generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY] is not None: init_example_dict = normalization.denormalize_data( new_example_dict=init_example_dict, training_example_dict=training_example_dict, normalization_type_string= generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY], min_normalized_value= generator_option_dict[neural_net.PREDICTOR_MIN_NORM_VALUE_KEY], max_normalized_value= generator_option_dict[neural_net.PREDICTOR_MAX_NORM_VALUE_KEY], separate_heights=True, apply_to_predictors=True, apply_to_vector_targets=False, apply_to_scalar_targets=False ) init_scalar_predictor_matrix = ( init_example_dict[example_utils.SCALAR_PREDICTOR_VALS_KEY] ) init_vector_predictor_matrix = ( init_example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] ) final_example_dict = copy.deepcopy(first_example_dict) this_example_dict = neural_net.predictors_numpy_to_dict( predictor_matrix=bwo_dict[bwo.FINAL_PREDICTORS_KEY], example_dict=final_example_dict, net_type_string=net_type_string ) final_example_dict.update(this_example_dict) if generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY] is not None: final_example_dict = normalization.denormalize_data( new_example_dict=final_example_dict, training_example_dict=training_example_dict, normalization_type_string= generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY], min_normalized_value= generator_option_dict[neural_net.PREDICTOR_MIN_NORM_VALUE_KEY], max_normalized_value= generator_option_dict[neural_net.PREDICTOR_MAX_NORM_VALUE_KEY], separate_heights=True, apply_to_predictors=True, apply_to_vector_targets=False, apply_to_scalar_targets=False ) final_scalar_predictor_matrix = ( final_example_dict[example_utils.SCALAR_PREDICTOR_VALS_KEY] ) final_vector_predictor_matrix = ( final_example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] ) print('Writing results to file: "{0:s}"...'.format(output_file_name)) bwo.write_file( netcdf_file_name=output_file_name, init_scalar_predictor_matrix=init_scalar_predictor_matrix, final_scalar_predictor_matrix=final_scalar_predictor_matrix, init_vector_predictor_matrix=init_vector_predictor_matrix, final_vector_predictor_matrix=final_vector_predictor_matrix, initial_activations=bwo_dict[bwo.INITIAL_ACTIVATIONS_KEY], final_activations=bwo_dict[bwo.FINAL_ACTIVATIONS_KEY], example_id_strings=example_id_strings, model_file_name=model_file_name, layer_name=layer_name, neuron_indices=neuron_indices, ideal_activation=ideal_activation, num_iterations=num_iterations, learning_rate=learning_rate, l2_weight=l2_weight )
def _run(net_type_string, training_dir_name, validation_dir_name, input_model_file_name, output_model_dir_name, use_generator_for_training, use_generator_for_validn, predictor_names, target_names, heights_m_agl, omit_heating_rate, first_training_time_string, last_training_time_string, first_validn_time_string, last_validn_time_string, normalization_file_name, predictor_norm_type_string, predictor_min_norm_value, predictor_max_norm_value, vector_target_norm_type_string, vector_target_min_norm_value, vector_target_max_norm_value, scalar_target_norm_type_string, scalar_target_min_norm_value, scalar_target_max_norm_value, num_examples_per_batch, num_epochs, num_training_batches_per_epoch, num_validn_batches_per_epoch, plateau_lr_multiplier): """Trains neural net :param net_type_string: See documentation at top of training_args.py. :param training_dir_name: Same. :param validation_dir_name: Same. :param input_model_file_name: Same. :param output_model_dir_name: Same. :param use_generator_for_training: Same. :param use_generator_for_validn: Same. :param predictor_names: Same. :param target_names: Same. :param heights_m_agl: Same. :param omit_heating_rate: Same. :param first_training_time_string: Same. :param last_training_time_string: Same. :param first_validn_time_string: Same. :param last_validn_time_string: Same. :param normalization_file_name: Same. :param predictor_norm_type_string: Same. :param predictor_min_norm_value: Same. :param predictor_max_norm_value: Same. :param vector_target_norm_type_string: Same. :param vector_target_min_norm_value: Same. :param vector_target_max_norm_value: Same. :param scalar_target_norm_type_string: Same. :param scalar_target_min_norm_value: Same. :param scalar_target_max_norm_value: Same. :param num_examples_per_batch: Same. :param num_epochs: Same. :param num_training_batches_per_epoch: Same. :param num_validn_batches_per_epoch: Same. :param plateau_lr_multiplier: Same. """ if predictor_norm_type_string in NONE_STRINGS: predictor_norm_type_string = None if vector_target_norm_type_string in NONE_STRINGS: vector_target_norm_type_string = None if scalar_target_norm_type_string in NONE_STRINGS: scalar_target_norm_type_string = None neural_net.check_net_type(net_type_string) if len(heights_m_agl) and heights_m_agl[0] <= 0: heights_m_agl = ( training_args.NET_TYPE_TO_DEFAULT_HEIGHTS_M_AGL[net_type_string]) for n in predictor_names: example_utils.check_field_name(n) scalar_predictor_names = [ n for n in predictor_names if n in example_utils.ALL_SCALAR_PREDICTOR_NAMES ] vector_predictor_names = [ n for n in predictor_names if n in example_utils.ALL_VECTOR_PREDICTOR_NAMES ] for n in target_names: example_utils.check_field_name(n) scalar_target_names = [ n for n in target_names if n in example_utils.ALL_SCALAR_TARGET_NAMES ] vector_target_names = [ n for n in target_names if n in example_utils.ALL_VECTOR_TARGET_NAMES ] first_training_time_unix_sec = time_conversion.string_to_unix_sec( first_training_time_string, training_args.TIME_FORMAT) last_training_time_unix_sec = time_conversion.string_to_unix_sec( last_training_time_string, training_args.TIME_FORMAT) first_validn_time_unix_sec = time_conversion.string_to_unix_sec( first_validn_time_string, training_args.TIME_FORMAT) last_validn_time_unix_sec = time_conversion.string_to_unix_sec( last_validn_time_string, training_args.TIME_FORMAT) training_option_dict = { neural_net.EXAMPLE_DIRECTORY_KEY: training_dir_name, neural_net.BATCH_SIZE_KEY: num_examples_per_batch, neural_net.SCALAR_PREDICTOR_NAMES_KEY: scalar_predictor_names, neural_net.VECTOR_PREDICTOR_NAMES_KEY: vector_predictor_names, neural_net.SCALAR_TARGET_NAMES_KEY: scalar_target_names, neural_net.VECTOR_TARGET_NAMES_KEY: vector_target_names, neural_net.HEIGHTS_KEY: heights_m_agl, neural_net.OMIT_HEATING_RATE_KEY: omit_heating_rate, neural_net.NORMALIZATION_FILE_KEY: normalization_file_name, neural_net.PREDICTOR_NORM_TYPE_KEY: predictor_norm_type_string, neural_net.PREDICTOR_MIN_NORM_VALUE_KEY: predictor_min_norm_value, neural_net.PREDICTOR_MAX_NORM_VALUE_KEY: predictor_max_norm_value, neural_net.VECTOR_TARGET_NORM_TYPE_KEY: vector_target_norm_type_string, neural_net.VECTOR_TARGET_MIN_VALUE_KEY: vector_target_min_norm_value, neural_net.VECTOR_TARGET_MAX_VALUE_KEY: vector_target_max_norm_value, neural_net.SCALAR_TARGET_NORM_TYPE_KEY: scalar_target_norm_type_string, neural_net.SCALAR_TARGET_MIN_VALUE_KEY: scalar_target_min_norm_value, neural_net.SCALAR_TARGET_MAX_VALUE_KEY: scalar_target_max_norm_value, neural_net.FIRST_TIME_KEY: first_training_time_unix_sec, neural_net.LAST_TIME_KEY: last_training_time_unix_sec, # neural_net.MIN_COLUMN_LWP_KEY: 0.05, # neural_net.MAX_COLUMN_LWP_KEY: 1e12 } validation_option_dict = { neural_net.EXAMPLE_DIRECTORY_KEY: validation_dir_name, neural_net.BATCH_SIZE_KEY: num_examples_per_batch, neural_net.FIRST_TIME_KEY: first_validn_time_unix_sec, neural_net.LAST_TIME_KEY: last_validn_time_unix_sec } if input_model_file_name in NONE_STRINGS: model_object = u_net_architecture.create_model( option_dict=DEFAULT_ARCHITECTURE_OPTION_DICT, vector_loss_function=DEFAULT_VECTOR_LOSS_FUNCTION, scalar_loss_function=DEFAULT_SCALAR_LOSS_FUNCTION, num_output_channels=1) loss_function_or_dict = { 'conv_output': DEFAULT_VECTOR_LOSS_FUNCTION, 'dense_output': DEFAULT_SCALAR_LOSS_FUNCTION } else: print('Reading untrained model from: "{0:s}"...'.format( input_model_file_name)) model_object = neural_net.read_model(input_model_file_name) input_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(input_model_file_name)[0]) print('Reading loss function(s) from: "{0:s}"...'.format( input_metafile_name)) loss_function_or_dict = neural_net.read_metafile(input_metafile_name)[ neural_net.LOSS_FUNCTION_OR_DICT_KEY] print(SEPARATOR_STRING) if use_generator_for_training: neural_net.train_model_with_generator( model_object=model_object, output_dir_name=output_model_dir_name, num_epochs=num_epochs, num_training_batches_per_epoch=num_training_batches_per_epoch, training_option_dict=training_option_dict, use_generator_for_validn=use_generator_for_validn, num_validation_batches_per_epoch=num_validn_batches_per_epoch, validation_option_dict=validation_option_dict, net_type_string=net_type_string, loss_function_or_dict=loss_function_or_dict, do_early_stopping=True, plateau_lr_multiplier=plateau_lr_multiplier) else: neural_net.train_model_sans_generator( model_object=model_object, output_dir_name=output_model_dir_name, num_epochs=num_epochs, training_option_dict=training_option_dict, validation_option_dict=validation_option_dict, net_type_string=net_type_string, loss_function_or_dict=loss_function_or_dict, do_early_stopping=True, plateau_lr_multiplier=plateau_lr_multiplier)
def read_file(netcdf_file_name): """Reads saliency maps from NetCDF file. :param netcdf_file_name: Path to input file. :return: saliency_dict: Dictionary with the following keys. saliency_dict['scalar_saliency_matrix']: See doc for `write_file`. saliency_dict['vector_saliency_matrix']: Same. saliency_dict['example_id_strings']: Same. saliency_dict['model_file_name']: Same. saliency_dict['layer_name']: Same. saliency_dict['neuron_indices']: Same. saliency_dict['ideal_activation']: Same. saliency_dict['target_field_name']: Same. saliency_dict['target_height_m_agl']: Same. """ dataset_object = netCDF4.Dataset(netcdf_file_name) saliency_dict = { EXAMPLE_IDS_KEY: [ str(id) for id in netCDF4.chartostring( dataset_object.variables[EXAMPLE_IDS_KEY][:]) ], MODEL_FILE_KEY: str(getattr(dataset_object, MODEL_FILE_KEY)), LAYER_NAME_KEY: str(getattr(dataset_object, LAYER_NAME_KEY)), NEURON_INDICES_KEY: numpy.array(getattr(dataset_object, NEURON_INDICES_KEY), dtype=int), IDEAL_ACTIVATION_KEY: getattr(dataset_object, IDEAL_ACTIVATION_KEY), TARGET_FIELD_KEY: getattr(dataset_object, TARGET_FIELD_KEY), TARGET_HEIGHT_KEY: getattr(dataset_object, TARGET_HEIGHT_KEY) } if numpy.isnan(saliency_dict[IDEAL_ACTIVATION_KEY]): saliency_dict[IDEAL_ACTIVATION_KEY] = None if saliency_dict[TARGET_FIELD_KEY] == '': saliency_dict[TARGET_FIELD_KEY] = None if saliency_dict[TARGET_HEIGHT_KEY] < 0: saliency_dict[TARGET_HEIGHT_KEY] = None num_examples = dataset_object.dimensions[EXAMPLE_DIMENSION_KEY].size num_scalar_predictors = ( dataset_object.dimensions[SCALAR_PREDICTOR_DIM_KEY].size) num_vector_predictors = ( dataset_object.dimensions[VECTOR_PREDICTOR_DIM_KEY].size) num_heights = dataset_object.dimensions[HEIGHT_DIMENSION_KEY].size if SCALAR_SALIENCY_KEY in dataset_object.variables: saliency_dict[SCALAR_SALIENCY_KEY] = ( dataset_object.variables[SCALAR_SALIENCY_KEY][:]) else: model_metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(saliency_dict[MODEL_FILE_KEY])[0]) model_metadata_dict = neural_net.read_metafile(model_metafile_name) net_type_string = model_metadata_dict[neural_net.NET_TYPE_KEY] if net_type_string == neural_net.DENSE_NET_TYPE_STRING: these_dim = (num_examples, num_scalar_predictors) else: these_dim = (num_examples, num_heights, num_scalar_predictors) saliency_dict[SCALAR_SALIENCY_KEY] = numpy.full(these_dim, 0.) if VECTOR_SALIENCY_KEY in dataset_object.variables: saliency_dict[VECTOR_SALIENCY_KEY] = ( dataset_object.variables[VECTOR_SALIENCY_KEY][:]) else: these_dim = (num_examples, num_heights, num_vector_predictors) saliency_dict[VECTOR_SALIENCY_KEY] = numpy.full(these_dim, 0.) dataset_object.close() return saliency_dict