def _run(model_file_name, top_example_dir_name, storm_metafile_name,
num_examples, output_file_name):
"""Creates dummy saliency map for each storm object.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param top_example_dir_name: Same.
:param storm_metafile_name: Same.
:param num_examples: Same.
:param output_file_name: Same.
"""
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
model_metafile_name = '{0:s}/model_metadata.p'.format(
os.path.split(model_file_name)[0])
print(
'Reading model metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None
print(
'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name))
full_storm_id_strings, storm_times_unix_sec = (
tracking_io.read_ids_and_times(storm_metafile_name))
print(SEPARATOR_STRING)
if 0 < num_examples < len(full_storm_id_strings):
full_storm_id_strings = full_storm_id_strings[:num_examples]
storm_times_unix_sec = storm_times_unix_sec[:num_examples]
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_storm_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=training_option_dict,
layer_operation_dicts=model_metadata_dict[cnn.LAYER_OPERATIONS_KEY])
print(SEPARATOR_STRING)
predictor_matrices = example_dict[testing_io.INPUT_MATRICES_KEY]
sounding_pressure_matrix_pa = (
example_dict[testing_io.SOUNDING_PRESSURES_KEY])
radar_matrix = predictor_matrices[0]
num_examples = radar_matrix.shape[0]
num_channels = radar_matrix.shape[-1]
num_spatial_dim = len(radar_matrix.shape) - 2
if num_spatial_dim == 2:
kernel_matrix = numpy.expand_dims(EDGE_DETECTOR_MATRIX_2D, axis=-1)
else:
kernel_matrix = numpy.expand_dims(EDGE_DETECTOR_MATRIX_3D, axis=-1)
kernel_matrix = numpy.repeat(kernel_matrix, num_channels, axis=-1)
kernel_matrix = numpy.expand_dims(kernel_matrix, axis=-1)
kernel_matrix = numpy.repeat(kernel_matrix, num_channels, axis=-1)
radar_saliency_matrix = numpy.full(radar_matrix.shape, numpy.nan)
for i in range(num_examples):
if numpy.mod(i, 10) == 0:
print((
'Have created dummy saliency map for {0:d} of {1:d} examples...'
).format(i, num_examples))
if num_spatial_dim == 2:
this_saliency_matrix = standalone_utils.do_2d_convolution(
feature_matrix=radar_matrix[i, ...],
kernel_matrix=kernel_matrix,
pad_edges=True,
stride_length_px=1)
else:
this_saliency_matrix = standalone_utils.do_3d_convolution(
feature_matrix=radar_matrix[i, ...],
kernel_matrix=kernel_matrix,
pad_edges=True,
stride_length_px=1)
radar_saliency_matrix[i, ...] = this_saliency_matrix[0, ...]
print('Have created dummy saliency map for all {0:d} examples!'.format(
num_examples))
print(SEPARATOR_STRING)
saliency_matrices = [
radar_saliency_matrix if k == 0 else predictor_matrices[k]
for k in range(len(predictor_matrices))
]
saliency_matrices = trainval_io.separate_shear_and_reflectivity(
list_of_input_matrices=saliency_matrices,
training_option_dict=training_option_dict)
denorm_predictor_matrices = trainval_io.separate_shear_and_reflectivity(
list_of_input_matrices=copy.deepcopy(predictor_matrices),
training_option_dict=training_option_dict)
print('Denormalizing model inputs...')
denorm_predictor_matrices = model_interpretation.denormalize_data(
list_of_input_matrices=denorm_predictor_matrices,
model_metadata_dict=model_metadata_dict)
print('Writing saliency maps to file: "{0:s}"...'.format(output_file_name))
saliency_metadata_dict = saliency_maps.check_metadata(
component_type_string=model_interpretation.CLASS_COMPONENT_TYPE_STRING,
target_class=1)
saliency_maps.write_standard_file(
pickle_file_name=output_file_name,
denorm_predictor_matrices=denorm_predictor_matrices,
saliency_matrices=saliency_matrices,
full_storm_id_strings=full_storm_id_strings,
storm_times_unix_sec=storm_times_unix_sec,
model_file_name=model_file_name,
metadata_dict=saliency_metadata_dict,
sounding_pressure_matrix_pa=sounding_pressure_matrix_pa)
def _run(prediction_file_name, top_example_dir_name, diff_colour_map_name,
num_examples, allow_whitespace, plot_panel_names, add_titles,
label_colour_bars, colour_bar_length, top_output_dir_name):
"""Plots one or more radar images and their upconvnet reconstructions.
This is effectively the main method.
:param prediction_file_name: See documentation at top of file.
:param top_example_dir_name: Same.
:param diff_colour_map_name: Same.
:param num_examples: Same.
:param allow_whitespace: Same.
:param plot_panel_names: Same.
:param add_titles: Same.
:param label_colour_bars: Same.
:param colour_bar_length: Same.
:param top_output_dir_name: Same.
"""
diff_colour_map_object = pyplot.get_cmap(diff_colour_map_name)
# Read data.
print('Reading reconstructed radar images from: "{0:s}"...'.format(
prediction_file_name
))
prediction_dict = upconvnet.read_predictions(prediction_file_name)
reconstructed_matrices = prediction_dict[
upconvnet.RECON_IMAGE_MATRICES_KEY]
full_storm_id_strings = prediction_dict[upconvnet.FULL_STORM_IDS_KEY]
storm_times_unix_sec = prediction_dict[upconvnet.STORM_TIMES_KEY]
if 0 < num_examples < len(full_storm_id_strings):
reconstructed_matrices = [
a[:num_examples, ...] for a in reconstructed_matrices
]
full_storm_id_strings = full_storm_id_strings[:num_examples]
storm_times_unix_sec = storm_times_unix_sec[:num_examples]
upconvnet_file_name = prediction_dict[upconvnet.UPCONVNET_FILE_KEY]
upconvnet_metafile_name = cnn.find_metafile(upconvnet_file_name)
print('Reading upconvnet metadata from: "{0:s}"...'.format(
upconvnet_metafile_name
))
upconvnet_metadata_dict = upconvnet.read_model_metadata(
upconvnet_metafile_name)
cnn_file_name = upconvnet_metadata_dict[upconvnet.CNN_FILE_KEY]
cnn_metafile_name = cnn.find_metafile(cnn_file_name)
print('Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name))
cnn_metadata_dict = cnn.read_model_metadata(cnn_metafile_name)
training_option_dict = cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
training_option_dict[trainval_io.NORMALIZATION_TYPE_KEY] = None
training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = None
training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY] = False
cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict
print(SEPARATOR_STRING)
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_storm_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=training_option_dict,
layer_operation_dicts=cnn_metadata_dict[cnn.LAYER_OPERATIONS_KEY]
)
print(SEPARATOR_STRING)
plot_examples.plot_examples(
list_of_predictor_matrices=example_dict[testing_io.INPUT_MATRICES_KEY],
model_metadata_dict=cnn_metadata_dict,
output_dir_name='{0:s}/actual_images'.format(top_output_dir_name),
pmm_flag=False, plot_soundings=False, plot_radar_diffs=False,
allow_whitespace=allow_whitespace, plot_panel_names=plot_panel_names,
add_titles=add_titles, label_colour_bars=label_colour_bars,
colour_bar_length=colour_bar_length,
full_storm_id_strings=full_storm_id_strings,
storm_times_unix_sec=storm_times_unix_sec)
print(SEPARATOR_STRING)
plot_examples.plot_examples(
list_of_predictor_matrices=reconstructed_matrices,
model_metadata_dict=cnn_metadata_dict,
output_dir_name=
'{0:s}/reconstructed_images'.format(top_output_dir_name),
pmm_flag=False, plot_soundings=False, plot_radar_diffs=False,
allow_whitespace=allow_whitespace, plot_panel_names=plot_panel_names,
add_titles=add_titles, label_colour_bars=label_colour_bars,
colour_bar_length=colour_bar_length,
full_storm_id_strings=full_storm_id_strings,
storm_times_unix_sec=storm_times_unix_sec)
print(SEPARATOR_STRING)
difference_matrices = [
r - a for r, a in zip(
reconstructed_matrices, example_dict[testing_io.INPUT_MATRICES_KEY]
)
]
plot_examples.plot_examples(
list_of_predictor_matrices=difference_matrices,
model_metadata_dict=cnn_metadata_dict,
output_dir_name='{0:s}/differences'.format(top_output_dir_name),
pmm_flag=False, plot_soundings=False, plot_radar_diffs=True,
diff_colour_map_object=diff_colour_map_object, max_diff_percentile=99.,
allow_whitespace=allow_whitespace, plot_panel_names=plot_panel_names,
add_titles=add_titles, label_colour_bars=label_colour_bars,
colour_bar_length=colour_bar_length,
full_storm_id_strings=full_storm_id_strings,
storm_times_unix_sec=storm_times_unix_sec)
def _run(model_file_name, layer_names, top_example_dir_name,
storm_metafile_name, num_examples, top_output_dir_name):
"""Evaluates CNN (convolutional neural net) predictions.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param layer_names: Same.
:param top_example_dir_name: Same.
:param storm_metafile_name: Same.
:param num_examples: Same.
:param top_output_dir_name: Same.
:raises: ValueError: if feature maps do not have 2 or 3 spatial dimensions.
"""
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = cnn.read_model(model_file_name)
model_metafile_name = '{0:s}/model_metadata.p'.format(
os.path.split(model_file_name)[0]
)
print('Reading model metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None
print('Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name))
full_id_strings, storm_times_unix_sec = tracking_io.read_ids_and_times(
storm_metafile_name)
print(SEPARATOR_STRING)
if 0 < num_examples < len(full_id_strings):
full_id_strings = full_id_strings[:num_examples]
storm_times_unix_sec = storm_times_unix_sec[:num_examples]
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=training_option_dict,
layer_operation_dicts=model_metadata_dict[cnn.LAYER_OPERATIONS_KEY]
)
print(SEPARATOR_STRING)
predictor_matrices = example_dict[testing_io.INPUT_MATRICES_KEY]
include_soundings = (
training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is not None
)
if include_soundings:
sounding_matrix = predictor_matrices[-1]
else:
sounding_matrix = None
num_layers = len(layer_names)
feature_matrix_by_layer = [numpy.array([])] * num_layers
for k in range(num_layers):
if model_metadata_dict[cnn.CONV_2D3D_KEY]:
if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]:
feature_matrix_by_layer[k] = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=predictor_matrices[0],
sounding_matrix=sounding_matrix,
return_features=True, feature_layer_name=layer_names[k]
)
else:
feature_matrix_by_layer[k] = cnn.apply_2d3d_cnn(
model_object=model_object,
reflectivity_matrix_dbz=predictor_matrices[0],
azimuthal_shear_matrix_s01=predictor_matrices[1],
sounding_matrix=sounding_matrix,
return_features=True, feature_layer_name=layer_names[k]
)
else:
feature_matrix_by_layer[k] = cnn.apply_2d_or_3d_cnn(
model_object=model_object,
radar_image_matrix=predictor_matrices[0],
sounding_matrix=sounding_matrix,
return_features=True, feature_layer_name=layer_names[k]
)
for k in range(num_layers):
this_output_dir_name = '{0:s}/{1:s}'.format(
top_output_dir_name, layer_names[k]
)
file_system_utils.mkdir_recursive_if_necessary(
directory_name=this_output_dir_name)
_plot_feature_maps_one_layer(
feature_matrix=feature_matrix_by_layer[k],
full_id_strings=full_id_strings,
storm_times_unix_sec=storm_times_unix_sec,
layer_name=layer_names[k],
output_dir_name=this_output_dir_name)
print(SEPARATOR_STRING)
def _run(model_file_name, target_class, target_layer_name,
top_example_dir_name, storm_metafile_name, num_examples,
randomize_weights, cascading_random, output_file_name):
"""Runs Grad-CAM (gradient-weighted class-activation maps).
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param target_class: Same.
:param target_layer_name: Same.
:param top_example_dir_name: Same.
:param storm_metafile_name: Same.
:param num_examples: Same.
:param randomize_weights: Same.
:param cascading_random: Same.
:param output_file_name: Same.
"""
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
# Read model and metadata.
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = cnn.read_model(model_file_name)
model_metafile_name = '{0:s}/model_metadata.p'.format(
os.path.split(model_file_name)[0])
print(
'Reading model metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None
output_dir_name, pathless_output_file_name = os.path.split(
output_file_name)
extensionless_output_file_name, output_file_extension = os.path.splitext(
pathless_output_file_name)
if randomize_weights:
conv_dense_layer_names = _find_conv_and_dense_layers(model_object)
conv_dense_layer_names.reverse()
num_sets = len(conv_dense_layer_names)
else:
conv_dense_layer_names = []
num_sets = 1
print(
'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name))
full_storm_id_strings, storm_times_unix_sec = (
tracking_io.read_ids_and_times(storm_metafile_name))
print(SEPARATOR_STRING)
if 0 < num_examples < len(full_storm_id_strings):
full_storm_id_strings = full_storm_id_strings[:num_examples]
storm_times_unix_sec = storm_times_unix_sec[:num_examples]
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_storm_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=training_option_dict,
layer_operation_dicts=model_metadata_dict[cnn.LAYER_OPERATIONS_KEY])
print(SEPARATOR_STRING)
predictor_matrices = example_dict[testing_io.INPUT_MATRICES_KEY]
sounding_pressure_matrix_pa = (
example_dict[testing_io.SOUNDING_PRESSURES_KEY])
print('Denormalizing model inputs...')
denorm_predictor_matrices = trainval_io.separate_shear_and_reflectivity(
list_of_input_matrices=copy.deepcopy(predictor_matrices),
training_option_dict=training_option_dict)
denorm_predictor_matrices = model_interpretation.denormalize_data(
list_of_input_matrices=denorm_predictor_matrices,
model_metadata_dict=model_metadata_dict)
print(SEPARATOR_STRING)
for k in range(num_sets):
if randomize_weights:
if cascading_random:
_reset_weights_in_layer(model_object=model_object,
layer_name=conv_dense_layer_names[k])
this_model_object = model_object
this_output_file_name = (
'{0:s}/{1:s}_cascading-random_{2:s}{3:s}').format(
output_dir_name, extensionless_output_file_name,
conv_dense_layer_names[k].replace('_', '-'),
output_file_extension)
else:
this_model_object = keras.models.Model.from_config(
model_object.get_config())
this_model_object.set_weights(model_object.get_weights())
_reset_weights_in_layer(model_object=this_model_object,
layer_name=conv_dense_layer_names[k])
this_output_file_name = '{0:s}/{1:s}_random_{2:s}{3:s}'.format(
output_dir_name, extensionless_output_file_name,
conv_dense_layer_names[k].replace('_', '-'),
output_file_extension)
else:
this_model_object = model_object
this_output_file_name = output_file_name
# print(K.eval(this_model_object.get_layer(name='dense_53').weights[0]))
these_cam_matrices, these_guided_cam_matrices = (
_run_gradcam_one_weight_set(
model_object=this_model_object,
target_class=target_class,
target_layer_name=target_layer_name,
predictor_matrices=predictor_matrices,
training_option_dict=training_option_dict))
print('Writing results to file: "{0:s}"...'.format(
this_output_file_name))
gradcam.write_standard_file(
pickle_file_name=this_output_file_name,
denorm_predictor_matrices=denorm_predictor_matrices,
cam_matrices=these_cam_matrices,
guided_cam_matrices=these_guided_cam_matrices,
full_storm_id_strings=full_storm_id_strings,
storm_times_unix_sec=storm_times_unix_sec,
model_file_name=model_file_name,
target_class=target_class,
target_layer_name=target_layer_name,
sounding_pressure_matrix_pa=sounding_pressure_matrix_pa)
print(SEPARATOR_STRING)
def _run(activation_file_name, num_examples, top_example_dir_name,
output_dir_name):
"""Plots inputs (radar images) for CNN flow chart.
This is effectively the main method.
:param activation_file_name: See documentation at top of file.
:param num_examples: Same.
:param top_example_dir_name: Same.
:param output_dir_name: Same.
:raises: ValueError: if activation file contains activations for more than
one model component.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name
)
print('Reading data from: "{0:s}"...'.format(activation_file_name))
activation_matrix, activation_metadata_dict = (
model_activation.read_file(activation_file_name)
)
num_model_components = activation_matrix.shape[1]
if num_model_components > 1:
error_string = (
'The file should contain activations for only one model '
'component, not {0:d}.'
).format(num_model_components)
raise TypeError(error_string)
full_storm_id_strings = (
activation_metadata_dict[model_activation.FULL_IDS_KEY]
)
storm_times_unix_sec = (
activation_metadata_dict[model_activation.STORM_TIMES_KEY]
)
model_file_name = (
activation_metadata_dict[model_activation.MODEL_FILE_NAME_KEY]
)
model_metafile_name = '{0:s}/model_metadata.p'.format(
os.path.split(model_file_name)[0]
)
print('Reading metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
training_option_dict[trainval_io.NORMALIZATION_TYPE_KEY] = None
training_option_dict[trainval_io.SAMPLING_FRACTIONS_KEY] = None
training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None
training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = RADAR_HEIGHTS_M_AGL
model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict
model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY][
trainval_io.UPSAMPLE_REFLECTIVITY_KEY
] = False
if 0 < num_examples < len(full_storm_id_strings):
full_storm_id_strings = full_storm_id_strings[:num_examples]
storm_times_unix_sec = storm_times_unix_sec[:num_examples]
print(SEPARATOR_STRING)
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_storm_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY],
layer_operation_dicts=model_metadata_dict[cnn.LAYER_OPERATIONS_KEY]
)
print(SEPARATOR_STRING)
radar_matrix = example_dict[testing_io.INPUT_MATRICES_KEY][0]
sounding_matrix = example_dict[testing_io.INPUT_MATRICES_KEY][-1]
sounding_pressure_matrix_pa = (
example_dict[testing_io.SOUNDING_PRESSURES_KEY]
)
num_examples = len(full_storm_id_strings)
for i in range(num_examples):
_plot_one_example(
radar_matrix=radar_matrix[[i], ...],
sounding_matrix=sounding_matrix[[i], ...],
sounding_pressures_pascals=sounding_pressure_matrix_pa[i, ...],
full_storm_id_string=full_storm_id_strings[i],
storm_time_unix_sec=storm_times_unix_sec[i],
model_metadata_dict=model_metadata_dict,
output_dir_name=output_dir_name
)
print('\n')
def _run(model_file_name, component_type_string, target_class, layer_name,
ideal_activation, neuron_indices, channel_index, top_example_dir_name,
storm_metafile_name, num_examples, randomize_weights,
cascading_random, output_file_name):
"""Computes saliency map for each storm object and each model component.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param component_type_string: Same.
:param target_class: Same.
:param layer_name: Same.
:param ideal_activation: Same.
:param neuron_indices: Same.
:param channel_index: Same.
:param top_example_dir_name: Same.
:param storm_metafile_name: Same.
:param num_examples: Same.
:param randomize_weights: Same.
:param cascading_random: Same.
:param output_file_name: Same.
"""
# Check input args.
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
model_interpretation.check_component_type(component_type_string)
# Read model and metadata.
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = cnn.read_model(model_file_name)
model_metafile_name = '{0:s}/model_metadata.p'.format(
os.path.split(model_file_name)[0])
print(
'Reading model metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None
output_dir_name, pathless_output_file_name = os.path.split(
output_file_name)
extensionless_output_file_name, output_file_extension = os.path.splitext(
pathless_output_file_name)
if randomize_weights:
conv_dense_layer_names = _find_conv_and_dense_layers(model_object)
conv_dense_layer_names.reverse()
num_sets = len(conv_dense_layer_names)
else:
conv_dense_layer_names = []
num_sets = 1
print(
'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name))
full_storm_id_strings, storm_times_unix_sec = (
tracking_io.read_ids_and_times(storm_metafile_name))
print(SEPARATOR_STRING)
if 0 < num_examples < len(full_storm_id_strings):
full_storm_id_strings = full_storm_id_strings[:num_examples]
storm_times_unix_sec = storm_times_unix_sec[:num_examples]
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_storm_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=training_option_dict,
layer_operation_dicts=model_metadata_dict[cnn.LAYER_OPERATIONS_KEY])
print(SEPARATOR_STRING)
predictor_matrices = example_dict[testing_io.INPUT_MATRICES_KEY]
sounding_pressure_matrix_pa = example_dict[
testing_io.SOUNDING_PRESSURES_KEY]
denorm_predictor_matrices = trainval_io.separate_shear_and_reflectivity(
list_of_input_matrices=copy.deepcopy(predictor_matrices),
training_option_dict=training_option_dict)
print('Denormalizing model inputs...')
denorm_predictor_matrices = model_interpretation.denormalize_data(
list_of_input_matrices=denorm_predictor_matrices,
model_metadata_dict=model_metadata_dict)
print(SEPARATOR_STRING)
for k in range(num_sets):
if randomize_weights:
if cascading_random:
_reset_weights_in_layer(model_object=model_object,
layer_name=conv_dense_layer_names[k])
this_model_object = model_object
this_output_file_name = (
'{0:s}/{1:s}_cascading-random_{2:s}{3:s}').format(
output_dir_name, extensionless_output_file_name,
conv_dense_layer_names[k].replace('_', '-'),
output_file_extension)
else:
this_model_object = keras.models.Model.from_config(
model_object.get_config())
this_model_object.set_weights(model_object.get_weights())
_reset_weights_in_layer(model_object=this_model_object,
layer_name=conv_dense_layer_names[k])
this_output_file_name = '{0:s}/{1:s}_random_{2:s}{3:s}'.format(
output_dir_name, extensionless_output_file_name,
conv_dense_layer_names[k].replace('_', '-'),
output_file_extension)
else:
this_model_object = model_object
this_output_file_name = output_file_name
# print(K.eval(this_model_object.get_layer(name='dense_3').weights[0]))
if component_type_string == CLASS_COMPONENT_TYPE_STRING:
print('Computing saliency maps for target class {0:d}...'.format(
target_class))
saliency_matrices = (
saliency_maps.get_saliency_maps_for_class_activation(
model_object=this_model_object,
target_class=target_class,
list_of_input_matrices=predictor_matrices))
elif component_type_string == NEURON_COMPONENT_TYPE_STRING:
print(
('Computing saliency maps for neuron {0:s} in layer "{1:s}"...'
).format(str(neuron_indices), layer_name))
saliency_matrices = (
saliency_maps.get_saliency_maps_for_neuron_activation(
model_object=this_model_object,
layer_name=layer_name,
neuron_indices=neuron_indices,
list_of_input_matrices=predictor_matrices,
ideal_activation=ideal_activation))
else:
print((
'Computing saliency maps for channel {0:d} in layer "{1:s}"...'
).format(channel_index, layer_name))
saliency_matrices = (
saliency_maps.get_saliency_maps_for_channel_activation(
model_object=this_model_object,
layer_name=layer_name,
channel_index=channel_index,
list_of_input_matrices=predictor_matrices,
stat_function_for_neuron_activations=K.max,
ideal_activation=ideal_activation))
saliency_matrices = trainval_io.separate_shear_and_reflectivity(
list_of_input_matrices=saliency_matrices,
training_option_dict=training_option_dict)
print('Writing saliency maps to file: "{0:s}"...'.format(
this_output_file_name))
saliency_metadata_dict = saliency_maps.check_metadata(
component_type_string=component_type_string,
target_class=target_class,
layer_name=layer_name,
ideal_activation=ideal_activation,
neuron_indices=neuron_indices,
channel_index=channel_index)
saliency_maps.write_standard_file(
pickle_file_name=this_output_file_name,
denorm_predictor_matrices=denorm_predictor_matrices,
saliency_matrices=saliency_matrices,
full_storm_id_strings=full_storm_id_strings,
storm_times_unix_sec=storm_times_unix_sec,
model_file_name=model_file_name,
metadata_dict=saliency_metadata_dict,
sounding_pressure_matrix_pa=sounding_pressure_matrix_pa)
def _read_one_example(top_example_dir_name, full_storm_id_string,
storm_time_unix_sec, source_name, radar_field_name,
include_sounding):
"""Reads one example (storm object).
T = number of input tensors to model
H_s = number of heights in sounding
:param top_example_dir_name: See documentation at top of file.
:param full_storm_id_string: Full storm ID.
:param storm_time_unix_sec: Valid time of storm.
:param source_name: Radar source (must be accepted by
`radar_utils.check_data_source`).
:param radar_field_name: Name of radar field (must be accepted by
`radar_utils.check_field_name`).
:param include_sounding: Boolean flag.
:return: predictor_matrices: length-T list of numpy arrays, where
the [i]th array is the [i]th input tensor to the model. The first axis
of each array has length = 1.
:return: model_metadata_dict: See doc for `cnn.write_model_metadata`.
:return: sounding_pressures_pa: length-H numpy array of pressures. If
soundings were not read, this is None.
"""
if source_name == radar_utils.GRIDRAD_SOURCE_ID:
num_radar_rows = NUM_GRIDRAD_ROWS
num_radar_columns = NUM_GRIDRAD_COLUMNS
else:
num_radar_rows = NUM_MYRORSS_ROWS
num_radar_columns = NUM_MYRORSS_COLUMNS
training_option_dict = dict()
training_option_dict[trainval_io.RADAR_FIELDS_KEY] = [radar_field_name]
training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = RADAR_HEIGHTS_M_AGL
training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = (
SOUNDING_FIELD_NAMES if include_sounding else None)
training_option_dict[trainval_io.SOUNDING_HEIGHTS_KEY] = (
SOUNDING_HEIGHTS_M_AGL)
training_option_dict[trainval_io.NUM_ROWS_KEY] = num_radar_rows
training_option_dict[trainval_io.NUM_COLUMNS_KEY] = num_radar_columns
training_option_dict[trainval_io.NORMALIZATION_TYPE_KEY] = None
training_option_dict[trainval_io.TARGET_NAME_KEY] = DUMMY_TARGET_NAME
training_option_dict[trainval_io.BINARIZE_TARGET_KEY] = False
training_option_dict[trainval_io.SAMPLING_FRACTIONS_KEY] = None
training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None
training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY] = False
model_metadata_dict = {
cnn.TRAINING_OPTION_DICT_KEY: training_option_dict,
cnn.LAYER_OPERATIONS_KEY: None,
}
print(MINOR_SEPARATOR_STRING)
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=[full_storm_id_string],
desired_times_unix_sec=numpy.array([storm_time_unix_sec], dtype=int),
option_dict=model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY],
layer_operation_dicts=None)
predictor_matrices = example_dict[testing_io.INPUT_MATRICES_KEY]
sounding_pressure_matrix_pa = example_dict[
testing_io.SOUNDING_PRESSURES_KEY]
if sounding_pressure_matrix_pa is None:
sounding_pressures_pa = None
else:
sounding_pressures_pa = sounding_pressure_matrix_pa[0, ...]
return predictor_matrices, model_metadata_dict, sounding_pressures_pa
def _run(model_file_name, top_example_dir_name, storm_metafile_name,
num_examples, do_backwards_test, separate_radar_heights,
num_bootstrap_reps, output_file_name):
"""Runs permutation test with specific examples (storm objects).
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param top_example_dir_name: Same.
:param storm_metafile_name: Same.
:param num_examples: Same.
:param do_backwards_test: Same.
:param separate_radar_heights: Same.
:param num_bootstrap_reps: Same.
:param output_file_name: Same.
"""
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = cnn.read_model(model_file_name)
metafile_name = cnn.find_metafile(model_file_name=model_file_name)
print('Reading metadata from: "{0:s}"...'.format(metafile_name))
cnn_metadata_dict = cnn.read_model_metadata(metafile_name)
training_option_dict = cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
print(
'Reading storm metadata from: "{0:s}"...'.format(storm_metafile_name))
full_storm_id_strings, storm_times_unix_sec = (
tracking_io.read_ids_and_times(storm_metafile_name))
print(SEPARATOR_STRING)
if 0 < num_examples < len(full_storm_id_strings):
numpy.random.seed(RANDOM_SEED)
good_indices = numpy.random.permutation(len(full_storm_id_strings))
good_indices = good_indices[:num_examples]
full_storm_id_strings = [
full_storm_id_strings[k] for k in good_indices
]
storm_times_unix_sec = storm_times_unix_sec[good_indices]
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_storm_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=training_option_dict,
layer_operation_dicts=cnn_metadata_dict[cnn.LAYER_OPERATIONS_KEY])
print(SEPARATOR_STRING)
predictor_matrices = example_dict[testing_io.INPUT_MATRICES_KEY]
target_values = example_dict[testing_io.TARGET_ARRAY_KEY]
correlation_matrix, predictor_names = correlation.get_pearson_correlations(
predictor_matrices=predictor_matrices,
cnn_metadata_dict=cnn_metadata_dict,
separate_radar_heights=separate_radar_heights)
print(SEPARATOR_STRING)
num_predictors = len(predictor_names)
for i in range(num_predictors):
for j in range(i, num_predictors):
print(('Pearson correlation between "{0:s}" and "{1:s}" = {2:.3f}'
).format(predictor_names[i], predictor_names[j],
correlation_matrix[i, j]))
print(SEPARATOR_STRING)
if do_backwards_test:
result_dict = permutation.run_backwards_test(
model_object=model_object,
predictor_matrices=predictor_matrices,
target_values=target_values,
cnn_metadata_dict=cnn_metadata_dict,
cost_function=permutation_utils.negative_auc_function,
separate_radar_heights=separate_radar_heights,
num_bootstrap_reps=num_bootstrap_reps)
else:
result_dict = permutation.run_forward_test(
model_object=model_object,
predictor_matrices=predictor_matrices,
target_values=target_values,
cnn_metadata_dict=cnn_metadata_dict,
cost_function=permutation_utils.negative_auc_function,
separate_radar_heights=separate_radar_heights,
num_bootstrap_reps=num_bootstrap_reps)
print(SEPARATOR_STRING)
result_dict[permutation_utils.MODEL_FILE_KEY] = model_file_name
result_dict[permutation_utils.TARGET_VALUES_KEY] = target_values
result_dict[permutation_utils.FULL_IDS_KEY] = full_storm_id_strings
result_dict[permutation_utils.STORM_TIMES_KEY] = storm_times_unix_sec
print('Writing results to: "{0:s}"...'.format(output_file_name))
permutation_utils.write_results(result_dict=result_dict,
pickle_file_name=output_file_name)
def _run(cnn_file_name, upconvnet_file_name, top_example_dir_name,
baseline_storm_metafile_name, trial_storm_metafile_name,
num_baseline_examples, num_trial_examples, num_novel_examples,
cnn_feature_layer_name, percent_variance_to_keep, output_file_name):
"""Runs novelty detection.
This is effectively the main method.
:param cnn_file_name: See documentation at top of file.
:param upconvnet_file_name: Same.
:param top_example_dir_name: Same.
:param baseline_storm_metafile_name: Same.
:param trial_storm_metafile_name: Same.
:param num_baseline_examples: Same.
:param num_trial_examples: Same.
:param num_novel_examples: Same.
:param cnn_feature_layer_name: Same.
:param percent_variance_to_keep: Same.
:param output_file_name: Same.
:raises: ValueError: if dimensions of first CNN input matrix != dimensions
of upconvnet output.
"""
print('Reading trained CNN from: "{0:s}"...'.format(cnn_file_name))
cnn_model_object = cnn.read_model(cnn_file_name)
print('Reading trained upconvnet from: "{0:s}"...'.format(
upconvnet_file_name))
upconvnet_model_object = cnn.read_model(upconvnet_file_name)
_check_dimensions(cnn_model_object=cnn_model_object,
upconvnet_model_object=upconvnet_model_object)
print('Reading metadata for baseline examples from: "{0:s}"...'.format(
baseline_storm_metafile_name))
baseline_full_id_strings, baseline_times_unix_sec = (
tracking_io.read_ids_and_times(baseline_storm_metafile_name))
print('Reading metadata for trial examples from: "{0:s}"...'.format(
trial_storm_metafile_name))
trial_full_id_strings, trial_times_unix_sec = (
tracking_io.read_ids_and_times(trial_storm_metafile_name))
this_dict = _filter_examples(
trial_full_id_strings=trial_full_id_strings,
trial_times_unix_sec=trial_times_unix_sec,
num_trial_examples=num_trial_examples,
baseline_full_id_strings=baseline_full_id_strings,
baseline_times_unix_sec=baseline_times_unix_sec,
num_baseline_examples=num_baseline_examples,
num_novel_examples=num_novel_examples)
trial_full_id_strings = this_dict[TRIAL_STORM_IDS_KEY]
trial_times_unix_sec = this_dict[TRIAL_STORM_TIMES_KEY]
baseline_full_id_strings = this_dict[BASELINE_STORM_IDS_KEY]
baseline_times_unix_sec = this_dict[BASELINE_STORM_TIMES_KEY]
num_novel_examples = this_dict[NUM_NOVEL_EXAMPLES_KEY]
cnn_metafile_name = '{0:s}/model_metadata.p'.format(
os.path.split(cnn_file_name)[0])
print('Reading CNN metadata from: "{0:s}"...'.format(cnn_metafile_name))
cnn_metadata_dict = cnn.read_model_metadata(cnn_metafile_name)
print(SEPARATOR_STRING)
baseline_predictor_matrices = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=baseline_full_id_strings,
desired_times_unix_sec=baseline_times_unix_sec,
option_dict=cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY],
layer_operation_dicts=cnn_metadata_dict[cnn.LAYER_OPERATIONS_KEY])[
testing_io.INPUT_MATRICES_KEY]
print(SEPARATOR_STRING)
trial_predictor_matrices = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=trial_full_id_strings,
desired_times_unix_sec=trial_times_unix_sec,
option_dict=cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY],
layer_operation_dicts=cnn_metadata_dict[cnn.LAYER_OPERATIONS_KEY])[
testing_io.INPUT_MATRICES_KEY]
print(SEPARATOR_STRING)
novelty_dict = novelty_detection.do_novelty_detection(
baseline_predictor_matrices=baseline_predictor_matrices,
trial_predictor_matrices=trial_predictor_matrices,
cnn_model_object=cnn_model_object,
cnn_feature_layer_name=cnn_feature_layer_name,
upconvnet_model_object=upconvnet_model_object,
num_novel_examples=num_novel_examples,
multipass=False,
percent_variance_to_keep=percent_variance_to_keep)
print(SEPARATOR_STRING)
print('Denormalizing inputs and outputs of novelty detection...')
cnn_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY][
trainval_io.SOUNDING_FIELDS_KEY] = None
novelty_dict[novelty_detection.BASELINE_MATRIX_KEY] = (
model_interpretation.denormalize_data(
list_of_input_matrices=baseline_predictor_matrices[[0]],
model_metadata_dict=cnn_metadata_dict))
novelty_dict[novelty_detection.TRIAL_MATRIX_KEY] = (
model_interpretation.denormalize_data(
list_of_input_matrices=trial_predictor_matrices[[0]],
model_metadata_dict=cnn_metadata_dict))
novelty_dict[novelty_detection.UPCONV_MATRIX_KEY] = (
model_interpretation.denormalize_data(
list_of_input_matrices=[
novelty_dict[novelty_detection.UPCONV_NORM_MATRIX_KEY]
],
model_metadata_dict=cnn_metadata_dict))[0]
novelty_dict.pop(novelty_detection.UPCONV_NORM_MATRIX_KEY)
novelty_dict[novelty_detection.UPCONV_SVD_MATRIX_KEY] = (
model_interpretation.denormalize_data(
list_of_input_matrices=[
novelty_dict[novelty_detection.UPCONV_NORM_SVD_MATRIX_KEY]
],
model_metadata_dict=cnn_metadata_dict))[0]
novelty_dict.pop(novelty_detection.UPCONV_NORM_SVD_MATRIX_KEY)
novelty_dict = novelty_detection.add_metadata(
novelty_dict=novelty_dict,
baseline_full_id_strings=baseline_full_id_strings,
baseline_times_unix_sec=baseline_times_unix_sec,
trial_full_id_strings=trial_full_id_strings,
trial_times_unix_sec=trial_times_unix_sec,
cnn_file_name=cnn_file_name,
upconvnet_file_name=upconvnet_file_name)
print('Writing results to: "{0:s}"...'.format(output_file_name))
novelty_detection.write_standard_file(novelty_dict=novelty_dict,
pickle_file_name=output_file_name)
def _run(model_file_name, init_function_name, storm_metafile_name,
num_examples, top_example_dir_name, component_type_string,
target_class, layer_name, neuron_indices, channel_index,
num_iterations, ideal_activation, learning_rate, l2_weight,
radar_constraint_weight, minmax_constraint_weight, output_file_name):
"""Runs backwards optimization on a trained CNN.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param init_function_name: Same.
:param storm_metafile_name: Same.
:param num_examples: Same.
:param top_example_dir_name: Same.
:param component_type_string: Same.
:param target_class: Same.
:param layer_name: Same.
:param neuron_indices: Same.
:param channel_index: Same.
:param num_iterations: Same.
:param ideal_activation: Same.
:param learning_rate: Same.
:param l2_weight: Same.
:param radar_constraint_weight: Same.
:param minmax_constraint_weight: Same.
:param output_file_name: Same.
"""
if l2_weight <= 0:
l2_weight = None
if radar_constraint_weight <= 0:
radar_constraint_weight = None
if minmax_constraint_weight <= 0:
minmax_constraint_weight = None
if ideal_activation <= 0:
ideal_activation = None
if init_function_name in ['', 'None']:
init_function_name = None
model_interpretation.check_component_type(component_type_string)
model_metafile_name = '{0:s}/model_metadata.p'.format(
os.path.split(model_file_name)[0])
print(
'Reading model metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
input_matrices = None
init_function = None
full_storm_id_strings = None
storm_times_unix_sec = None
sounding_pressure_matrix_pa = None
if init_function_name is None:
print('Reading storm metadata from: "{0:s}"...'.format(
storm_metafile_name))
full_storm_id_strings, storm_times_unix_sec = (
tracking_io.read_ids_and_times(storm_metafile_name))
if 0 < num_examples < len(full_storm_id_strings):
full_storm_id_strings = full_storm_id_strings[:num_examples]
storm_times_unix_sec = storm_times_unix_sec[:num_examples]
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_storm_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY],
layer_operation_dicts=model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY])
print(SEPARATOR_STRING)
input_matrices = example_dict[testing_io.INPUT_MATRICES_KEY]
sounding_pressure_matrix_pa = example_dict[
testing_io.SOUNDING_PRESSURES_KEY]
num_examples = input_matrices[0].shape[0]
else:
num_examples = 1
init_function = _create_initializer(
init_function_name=init_function_name,
model_metadata_dict=model_metadata_dict)
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = cnn.read_model(model_file_name)
output_matrices = None
initial_activations = numpy.full(num_examples, numpy.nan)
final_activations = numpy.full(num_examples, numpy.nan)
for i in range(num_examples):
if init_function_name is None:
this_init_arg = [a[[i], ...] for a in input_matrices]
else:
this_init_arg = init_function
if component_type_string == CLASS_COMPONENT_TYPE_STRING:
print((
'\nOptimizing {0:d}th of {1:d} images for target class {2:d}...'
).format(i + 1, num_examples, target_class))
this_result_dict = backwards_opt.optimize_input_for_class(
model_object=model_object,
target_class=target_class,
init_function_or_matrices=this_init_arg,
num_iterations=num_iterations,
learning_rate=learning_rate,
l2_weight=l2_weight,
radar_constraint_weight=radar_constraint_weight,
minmax_constraint_weight=minmax_constraint_weight,
model_metadata_dict=model_metadata_dict)
elif component_type_string == NEURON_COMPONENT_TYPE_STRING:
print((
'\nOptimizing {0:d}th of {1:d} images for neuron {2:s} in layer'
' "{3:s}"...').format(i + 1, num_examples, str(neuron_indices),
layer_name))
this_result_dict = backwards_opt.optimize_input_for_neuron(
model_object=model_object,
layer_name=layer_name,
neuron_indices=neuron_indices,
init_function_or_matrices=this_init_arg,
num_iterations=num_iterations,
learning_rate=learning_rate,
l2_weight=l2_weight,
ideal_activation=ideal_activation,
radar_constraint_weight=radar_constraint_weight,
minmax_constraint_weight=minmax_constraint_weight,
model_metadata_dict=model_metadata_dict)
else:
print(('\nOptimizing {0:d}th of {1:d} images for channel {2:d} in '
'layer "{3:s}"...').format(i + 1, num_examples,
channel_index, layer_name))
this_result_dict = backwards_opt.optimize_input_for_channel(
model_object=model_object,
layer_name=layer_name,
channel_index=channel_index,
init_function_or_matrices=this_init_arg,
stat_function_for_neuron_activations=K.max,
num_iterations=num_iterations,
learning_rate=learning_rate,
l2_weight=l2_weight,
ideal_activation=ideal_activation,
radar_constraint_weight=radar_constraint_weight,
minmax_constraint_weight=minmax_constraint_weight,
model_metadata_dict=model_metadata_dict)
initial_activations[i] = this_result_dict[
backwards_opt.INITIAL_ACTIVATION_KEY]
final_activations[i] = this_result_dict[
backwards_opt.FINAL_ACTIVATION_KEY]
these_output_matrices = this_result_dict[
backwards_opt.NORM_OUTPUT_MATRICES_KEY]
if output_matrices is None:
output_matrices = [None] * len(these_output_matrices)
for k in range(len(output_matrices)):
if output_matrices[k] is None:
output_matrices[k] = these_output_matrices[k] + 0.
else:
output_matrices[k] = numpy.concatenate(
(output_matrices[k], these_output_matrices[k]), axis=0)
if init_function_name is None:
continue
these_input_matrices = this_result_dict[
backwards_opt.NORM_INPUT_MATRICES_KEY]
if input_matrices is None:
input_matrices = [None] * len(these_input_matrices)
for k in range(len(input_matrices)):
if input_matrices[k] is None:
input_matrices[k] = these_input_matrices[k] + 0.
else:
input_matrices[k] = numpy.concatenate(
(input_matrices[k], these_input_matrices[k]), axis=0)
print(SEPARATOR_STRING)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
print('Denormalizing input examples...')
input_matrices = trainval_io.separate_shear_and_reflectivity(
list_of_input_matrices=input_matrices,
training_option_dict=training_option_dict)
input_matrices = model_interpretation.denormalize_data(
list_of_input_matrices=input_matrices,
model_metadata_dict=model_metadata_dict)
print('Denormalizing optimized examples...')
output_matrices = trainval_io.separate_shear_and_reflectivity(
list_of_input_matrices=output_matrices,
training_option_dict=training_option_dict)
output_matrices = model_interpretation.denormalize_data(
list_of_input_matrices=output_matrices,
model_metadata_dict=model_metadata_dict)
print('Writing results to: "{0:s}"...'.format(output_file_name))
bwo_metadata_dict = backwards_opt.check_metadata(
component_type_string=component_type_string,
num_iterations=num_iterations,
learning_rate=learning_rate,
target_class=target_class,
layer_name=layer_name,
ideal_activation=ideal_activation,
neuron_indices=neuron_indices,
channel_index=channel_index,
l2_weight=l2_weight,
radar_constraint_weight=radar_constraint_weight,
minmax_constraint_weight=minmax_constraint_weight)
backwards_opt.write_standard_file(
pickle_file_name=output_file_name,
denorm_input_matrices=input_matrices,
denorm_output_matrices=output_matrices,
initial_activations=initial_activations,
final_activations=final_activations,
model_file_name=model_file_name,
metadata_dict=bwo_metadata_dict,
full_storm_id_strings=full_storm_id_strings,
storm_times_unix_sec=storm_times_unix_sec,
sounding_pressure_matrix_pa=sounding_pressure_matrix_pa)
def _run(activation_file_name, storm_metafile_name, num_examples,
top_example_dir_name, num_radar_rows, num_radar_columns,
allow_whitespace, colour_bar_length, output_dir_name):
"""Plots one or more examples (storm objects) for human input.
This is effectively the main method.
:param activation_file_name: See documentation at top of file.
:param storm_metafile_name: Same.
:param num_examples: Same.
:param top_example_dir_name: Same.
:param num_radar_rows: Same.
:param num_radar_columns: Same.
:param allow_whitespace: Same.
:param colour_bar_length: Same.
:param output_dir_name: Same.
"""
if num_radar_rows <= 0:
num_radar_rows = None
if num_radar_columns <= 0:
num_radar_columns = None
if activation_file_name in ['', 'None']:
activation_file_name = None
if activation_file_name is None:
print('Reading data from: "{0:s}"...'.format(storm_metafile_name))
full_storm_id_strings, storm_times_unix_sec = (
tracking_io.read_ids_and_times(storm_metafile_name))
training_option_dict = dict()
training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] = None
training_option_dict[trainval_io.SOUNDING_HEIGHTS_KEY] = None
training_option_dict[trainval_io.NUM_ROWS_KEY] = num_radar_rows
training_option_dict[trainval_io.NUM_COLUMNS_KEY] = num_radar_columns
training_option_dict[trainval_io.NORMALIZATION_TYPE_KEY] = None
training_option_dict[trainval_io.TARGET_NAME_KEY] = DUMMY_TARGET_NAME
training_option_dict[trainval_io.BINARIZE_TARGET_KEY] = False
training_option_dict[trainval_io.SAMPLING_FRACTIONS_KEY] = None
training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None
model_metadata_dict = {cnn.LAYER_OPERATIONS_KEY: None}
else:
print('Reading data from: "{0:s}"...'.format(activation_file_name))
activation_matrix, activation_metadata_dict = (
model_activation.read_file(activation_file_name))
num_model_components = activation_matrix.shape[1]
if num_model_components > 1:
error_string = (
'The file should contain activations for only one model '
'component, not {0:d}.').format(num_model_components)
raise TypeError(error_string)
full_storm_id_strings = activation_metadata_dict[
model_activation.FULL_IDS_KEY]
storm_times_unix_sec = activation_metadata_dict[
model_activation.STORM_TIMES_KEY]
model_file_name = activation_metadata_dict[
model_activation.MODEL_FILE_NAME_KEY]
model_metafile_name = '{0:s}/model_metadata.p'.format(
os.path.split(model_file_name)[0])
print('Reading metadata from: "{0:s}"...'.format(model_metafile_name))
model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
training_option_dict = model_metadata_dict[
cnn.TRAINING_OPTION_DICT_KEY]
training_option_dict[trainval_io.NORMALIZATION_TYPE_KEY] = None
training_option_dict[trainval_io.SAMPLING_FRACTIONS_KEY] = None
training_option_dict[trainval_io.REFLECTIVITY_MASK_KEY] = None
training_option_dict[trainval_io.RADAR_FIELDS_KEY] = SHEAR_FIELD_NAMES
training_option_dict[trainval_io.RADAR_HEIGHTS_KEY] = REFL_HEIGHTS_M_AGL
training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY] = False
model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY] = training_option_dict
if 0 < num_examples < len(full_storm_id_strings):
full_storm_id_strings = full_storm_id_strings[:num_examples]
storm_times_unix_sec = storm_times_unix_sec[:num_examples]
print(SEPARATOR_STRING)
example_dict = testing_io.read_predictors_specific_examples(
top_example_dir_name=top_example_dir_name,
desired_full_id_strings=full_storm_id_strings,
desired_times_unix_sec=storm_times_unix_sec,
option_dict=model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY],
layer_operation_dicts=model_metadata_dict[cnn.LAYER_OPERATIONS_KEY])
print(SEPARATOR_STRING)
predictor_matrices = example_dict[testing_io.INPUT_MATRICES_KEY]
# TODO(thunderhoser): The rest of this code is very HACKY.
predictor_matrices[0] = trainval_io.upsample_reflectivity(
predictor_matrices[0][..., 0])
predictor_matrices[0] = numpy.expand_dims(predictor_matrices[0], axis=-1)
example_dict = {
input_examples.RADAR_FIELDS_KEY: SHEAR_FIELD_NAMES,
input_examples.REFL_IMAGE_MATRIX_KEY: predictor_matrices[0],
input_examples.AZ_SHEAR_IMAGE_MATRIX_KEY: predictor_matrices[1],
input_examples.RADAR_HEIGHTS_KEY: REFL_HEIGHTS_M_AGL
}
example_dict = input_examples.reduce_examples_3d_to_2d(
example_dict=example_dict,
list_of_operation_dicts=[REFL_LAYER_OPERATION_DICT])
predictor_matrices = [example_dict[input_examples.RADAR_IMAGE_MATRIX_KEY]]
layer_operation_dicts = [{
input_examples.RADAR_FIELD_KEY: f,
input_examples.MIN_HEIGHT_KEY: h1,
input_examples.MAX_HEIGHT_KEY: h2,
input_examples.OPERATION_NAME_KEY: op
} for f, h1, h2, op in zip(
example_dict[input_examples.RADAR_FIELDS_KEY], example_dict[
input_examples.MIN_RADAR_HEIGHTS_KEY], example_dict[
input_examples.MAX_RADAR_HEIGHTS_KEY], example_dict[
input_examples.RADAR_LAYER_OPERATION_NAMES_KEY])]
model_metadata_dict[cnn.LAYER_OPERATIONS_KEY] = layer_operation_dicts
figure_file_names = plot_examples.plot_examples(
list_of_predictor_matrices=predictor_matrices,
model_metadata_dict=model_metadata_dict,
pmm_flag=False,
output_dir_name=output_dir_name,
plot_soundings=False,
allow_whitespace=allow_whitespace,
plot_panel_names=False,
add_titles=False,
label_colour_bars=True,
colour_bar_length=colour_bar_length,
colour_bar_font_size=COLOUR_BAR_FONT_SIZE,
figure_resolution_dpi=FIGURE_RESOLUTION_DPI,
refl_opacity=REFL_OPACITY,
plot_grid_lines=False,
full_storm_id_strings=full_storm_id_strings,
storm_times_unix_sec=storm_times_unix_sec)
for this_file_name in figure_file_names:
print('Resizing image to {0:d} pixels: "{1:s}"...'.format(
FIGURE_SIZE_PIXELS, this_file_name))
imagemagick_utils.resize_image(input_file_name=this_file_name,
output_file_name=this_file_name,
output_size_pixels=FIGURE_SIZE_PIXELS)