def test_radar_fields_and_heights_to_panel_names_sans_units(self):
"""Ensures correctness of radar_fields_and_heights_to_panel_names.
In this case, expecting panel names without units.
"""
these_panel_names = (
radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=FIELD_NAME_BY_PAIR,
heights_m_agl=HEIGHT_BY_PAIR_M_AGL,
include_units=False))
self.assertTrue(these_panel_names == PANEL_NAMES_SANS_OPS_SANS_UNITS)
def _plot_2d_radar(model_metadata_dict,
output_dir_name,
pmm_flag,
diff_colour_map_object=None,
max_colour_percentile_for_diff=None,
full_id_strings=None,
storm_time_strings=None,
novel_radar_matrix=None,
novel_radar_matrix_upconv=None,
novel_radar_matrix_upconv_svd=None):
"""Plots results of novelty detection for 2-D radar fields.
E = number of examples (storm objects)
M = number of rows in spatial grid
N = number of columns in spatial grid
C = number of channels (field/height pairs)
This method handles the 3 input matrices in the same way as
`_plot_3d_radar`.
:param model_metadata_dict: Dictionary returned by
`cnn.read_model_metadata`.
:param output_dir_name: Same.
:param pmm_flag: Same.
:param diff_colour_map_object: Same.
:param max_colour_percentile_for_diff: Same.
:param full_id_strings: Same.
:param storm_time_strings: Same.
:param novel_radar_matrix: E-by-M-by-N-by-C numpy array of original
(not reconstructed) radar fields.
:param novel_radar_matrix_upconv: E-by-M-by-N-by-C numpy array of
upconvnet-reconstructed radar fields.
:param novel_radar_matrix_upconv_svd: E-by-M-by-N-by-C numpy array of
upconvnet-and-SVD-reconstructed radar fields.
"""
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if pmm_flag:
have_storm_ids = False
else:
have_storm_ids = not (full_id_strings is None
or storm_time_strings is None)
plot_difference = False
if novel_radar_matrix is not None:
plot_type_abbrev = 'actual'
plot_type_verbose = 'actual'
radar_matrix_to_plot = novel_radar_matrix
else:
if (novel_radar_matrix_upconv is not None
and novel_radar_matrix_upconv_svd is not None):
plot_difference = True
plot_type_abbrev = 'novelty'
plot_type_verbose = 'novelty'
radar_matrix_to_plot = (novel_radar_matrix_upconv -
novel_radar_matrix_upconv_svd)
else:
if novel_radar_matrix_upconv is not None:
plot_type_abbrev = 'upconv'
plot_type_verbose = 'upconvnet reconstruction'
radar_matrix_to_plot = novel_radar_matrix_upconv
else:
plot_type_abbrev = 'upconv-svd'
plot_type_verbose = 'upconvnet/SVD reconstruction'
radar_matrix_to_plot = novel_radar_matrix_upconv_svd
if list_of_layer_operation_dicts is None:
field_name_by_panel = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
panel_names = (radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=field_name_by_panel,
heights_m_agl=training_option_dict[trainval_io.RADAR_HEIGHTS_KEY]))
plot_colour_bar_by_panel = numpy.full(len(panel_names),
True,
dtype=bool)
else:
field_name_by_panel, panel_names = (
radar_plotting.layer_ops_to_field_and_panel_names(
list_of_layer_operation_dicts))
plot_colour_bar_by_panel = numpy.full(len(panel_names),
False,
dtype=bool)
plot_colour_bar_by_panel[2::3] = True
num_panels = len(panel_names)
num_storms = radar_matrix_to_plot.shape[0]
num_channels = radar_matrix_to_plot.shape[-1]
num_panel_rows = int(numpy.floor(numpy.sqrt(num_channels)))
for i in range(num_storms):
if pmm_flag:
this_title_string = 'Probability-matched mean'
this_file_name = 'pmm'
else:
if have_storm_ids:
this_title_string = 'Storm "{0:s}" at {1:s}'.format(
full_id_strings[i], storm_time_strings[i])
this_file_name = '{0:s}_{1:s}'.format(
full_id_strings[i].replace('_', '-'),
storm_time_strings[i])
else:
this_title_string = 'Example {0:d}'.format(i + 1)
this_file_name = 'example{0:06d}'.format(i)
this_title_string += ' ({0:s})'.format(plot_type_verbose)
this_file_name = '{0:s}/{1:s}_{2:s}_radar.jpg'.format(
output_dir_name, this_file_name, plot_type_abbrev)
if plot_difference:
this_cmap_object_by_panel = [diff_colour_map_object] * num_panels
this_cnorm_object_by_panel = [None] * num_panels
if list_of_layer_operation_dicts is None:
for j in range(num_panels):
this_max_value = numpy.percentile(
numpy.absolute(radar_matrix_to_plot[i, ..., j]),
max_colour_percentile_for_diff)
this_cnorm_object_by_panel[
j] = matplotlib.colors.Normalize(vmin=-1 *
this_max_value,
vmax=this_max_value,
clip=False)
else:
unique_field_names = numpy.unique(
numpy.array(field_name_by_panel))
for this_field_name in unique_field_names:
these_panel_indices = numpy.where(
numpy.array(field_name_by_panel) == this_field_name)[0]
this_diff_matrix = radar_matrix_to_plot[
i, ..., these_panel_indices]
this_max_value = numpy.percentile(
numpy.absolute(this_diff_matrix),
max_colour_percentile_for_diff)
for this_index in these_panel_indices:
this_cnorm_object_by_panel[this_index] = (
matplotlib.colors.Normalize(vmin=-1 *
this_max_value,
vmax=this_max_value,
clip=False))
else:
this_cmap_object_by_panel = None
this_cnorm_object_by_panel = None
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(radar_matrix_to_plot[i, ...], axis=0),
field_name_by_panel=field_name_by_panel,
num_panel_rows=num_panel_rows,
panel_names=panel_names,
colour_map_object_by_panel=this_cmap_object_by_panel,
colour_norm_object_by_panel=this_cnorm_object_by_panel,
plot_colour_bar_by_panel=plot_colour_bar_by_panel,
font_size=FONT_SIZE_WITH_COLOUR_BARS,
row_major=False)
pyplot.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
print('Saving figure to: "{0:s}"...'.format(this_file_name))
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
def _plot_saliency_for_2d_radar(
radar_matrix, radar_saliency_matrix, model_metadata_dict,
saliency_colour_map_object, max_colour_value_by_example,
output_dir_name, storm_ids=None, storm_times_unix_sec=None):
"""Plots saliency for 2-D radar fields.
E = number of examples
M = number of rows in spatial grid
N = number of columns in spatial grid
C = number of channels (field/height pairs)
If `storm_ids is None` and `storm_times_unix_sec is None`, will assume that
the input matrices contain probability-matched means.
:param radar_matrix: E-by-M-by-N-by-C numpy array of radar values
(predictors).
:param radar_saliency_matrix: E-by-M-by-N-by-C numpy array of saliency
values.
:param model_metadata_dict: See doc for `cnn.read_model_metadata`.
:param saliency_colour_map_object: See doc for
`_plot_saliency_for_2d3d_radar`.
:param max_colour_value_by_example: Same.
:param output_dir_name: Same.
:param storm_ids: Same.
:param storm_times_unix_sec: Same.
"""
pmm_flag = storm_ids is None and storm_times_unix_sec is None
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if list_of_layer_operation_dicts is None:
field_name_by_panel = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
panel_names = (
radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=field_name_by_panel,
heights_m_agl=training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]
)
)
plot_colour_bar_by_panel = numpy.full(
len(panel_names), True, dtype=bool)
else:
field_name_by_panel, panel_names = (
radar_plotting.layer_ops_to_field_and_panel_names(
list_of_layer_operation_dicts)
)
plot_colour_bar_by_panel = numpy.full(
len(panel_names), False, dtype=bool)
plot_colour_bar_by_panel[2::3] = True
num_examples = radar_matrix.shape[0]
num_panels = len(field_name_by_panel)
num_panel_rows = int(numpy.floor(numpy.sqrt(num_panels)))
for i in range(num_examples):
_, these_axes_objects = (
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(radar_matrix[i, ...], axis=0),
field_name_by_panel=field_name_by_panel,
panel_names=panel_names, num_panel_rows=num_panel_rows,
plot_colour_bar_by_panel=plot_colour_bar_by_panel,
font_size=FONT_SIZE_WITH_COLOUR_BARS, row_major=False)
)
this_contour_interval = (
max_colour_value_by_example[i] / HALF_NUM_CONTOURS
)
saliency_plotting.plot_many_2d_grids_with_contours(
saliency_matrix_3d=numpy.flip(
radar_saliency_matrix[i, ...], axis=0),
axes_objects_2d_list=these_axes_objects,
colour_map_object=saliency_colour_map_object,
max_absolute_contour_level=max_colour_value_by_example[i],
contour_interval=this_contour_interval, row_major=False)
if pmm_flag:
this_title_string = 'Probability-matched mean'
this_file_name = '{0:s}/saliency_pmm_radar.jpg'.format(
output_dir_name)
else:
this_storm_time_string = time_conversion.unix_sec_to_string(
storm_times_unix_sec[i], TIME_FORMAT)
this_title_string = 'Storm "{0:s}" at {1:s}'.format(
storm_ids[i], this_storm_time_string)
this_file_name = '{0:s}/saliency_{1:s}_{2:s}_radar.jpg'.format(
output_dir_name, storm_ids[i].replace('_', '-'),
this_storm_time_string)
this_title_string += ' (max absolute saliency = {0:.3f})'.format(
max_colour_value_by_example[i])
pyplot.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
print 'Saving figure to file: "{0:s}"...'.format(this_file_name)
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
def _plot_bwo_for_2d_radar(
optimized_radar_matrix, model_metadata_dict, diff_colour_map_object,
max_colour_percentile_for_diff, top_output_dir_name, pmm_flag,
input_radar_matrix=None, storm_ids=None, storm_times_unix_sec=None):
"""Plots BWO results for 2-D radar fields.
E = number of examples (storm objects)
M = number of rows in spatial grid
N = number of columns in spatial grid
C = number of channels (field/height pairs)
:param optimized_radar_matrix: E-by-M-by-N-by-C numpy array of radar values
(predictors).
:param model_metadata_dict: Dictionary returned by
`cnn.read_model_metadata`.
:param diff_colour_map_object: See doc for `_plot_bwo_for_2d3d_radar`.
:param max_colour_percentile_for_diff: Same.
:param top_output_dir_name: Same.
:param pmm_flag: Same.
:param input_radar_matrix: Same as `optimized_radar_matrix` but with
non-optimized input.
:param storm_ids: See doc for `_plot_bwo_for_2d3d_radar`.
:param storm_times_unix_sec: Same.
"""
before_optimization_dir_name = '{0:s}/before_optimization'.format(
top_output_dir_name)
after_optimization_dir_name = '{0:s}/after_optimization'.format(
top_output_dir_name)
difference_dir_name = '{0:s}/after_minus_before_optimization'.format(
top_output_dir_name)
file_system_utils.mkdir_recursive_if_necessary(
directory_name=before_optimization_dir_name)
file_system_utils.mkdir_recursive_if_necessary(
directory_name=after_optimization_dir_name)
file_system_utils.mkdir_recursive_if_necessary(
directory_name=difference_dir_name)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if pmm_flag:
have_storm_ids = False
else:
have_storm_ids = not (storm_ids is None or storm_times_unix_sec is None)
if list_of_layer_operation_dicts is None:
field_name_by_panel = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
panel_names = (
radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=field_name_by_panel,
heights_m_agl=training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]
)
)
plot_colour_bar_by_panel = numpy.full(
len(panel_names), True, dtype=bool)
else:
field_name_by_panel, panel_names = (
radar_plotting.layer_ops_to_field_and_panel_names(
list_of_layer_operation_dicts)
)
plot_colour_bar_by_panel = numpy.full(
len(panel_names), False, dtype=bool)
plot_colour_bar_by_panel[2::3] = True
num_panels = len(panel_names)
num_storms = optimized_radar_matrix.shape[0]
num_channels = optimized_radar_matrix.shape[-1]
num_panel_rows = int(numpy.floor(
numpy.sqrt(num_channels)
))
for i in range(num_storms):
print '\n'
if pmm_flag:
this_base_title_string = 'Probability-matched mean'
this_base_pathless_file_name = 'pmm'
else:
if have_storm_ids:
this_storm_time_string = time_conversion.unix_sec_to_string(
storm_times_unix_sec[i], TIME_FORMAT)
this_base_title_string = 'Storm "{0:s}" at {1:s}'.format(
storm_ids[i], this_storm_time_string)
this_base_pathless_file_name = '{0:s}_{1:s}'.format(
storm_ids[i].replace('_', '-'), this_storm_time_string)
else:
this_base_title_string = 'Example {0:d}'.format(i + 1)
this_base_pathless_file_name = 'example{0:06d}'.format(i)
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(optimized_radar_matrix[i, ...], axis=0),
field_name_by_panel=field_name_by_panel,
num_panel_rows=num_panel_rows, panel_names=panel_names,
plot_colour_bar_by_panel=plot_colour_bar_by_panel,
font_size=FONT_SIZE_WITH_COLOUR_BARS, row_major=False)
this_title_string = '{0:s} (after optimization)'.format(
this_base_title_string)
this_file_name = '{0:s}/{1:s}_after-optimization_radar.jpg'.format(
after_optimization_dir_name, this_base_pathless_file_name)
pyplot.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
print 'Saving figure to: "{0:s}"...'.format(this_file_name)
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
if input_radar_matrix is None:
continue
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(input_radar_matrix[i, ...], axis=0),
field_name_by_panel=field_name_by_panel,
num_panel_rows=num_panel_rows, panel_names=panel_names,
plot_colour_bar_by_panel=plot_colour_bar_by_panel,
font_size=FONT_SIZE_WITH_COLOUR_BARS, row_major=False)
this_title_string = '{0:s} (before optimization)'.format(
this_base_title_string)
this_file_name = '{0:s}/{1:s}_before-optimization_radar.jpg'.format(
before_optimization_dir_name, this_base_pathless_file_name)
pyplot.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
print 'Saving figure to: "{0:s}"...'.format(this_file_name)
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
this_cmap_object_by_panel = [diff_colour_map_object] * num_panels
this_cnorm_object_by_panel = [None] * num_panels
if list_of_layer_operation_dicts is None:
for j in range(num_panels):
this_diff_matrix = (
optimized_radar_matrix[i, ..., j] -
input_radar_matrix[i, ..., j]
)
this_max_value = numpy.percentile(
numpy.absolute(this_diff_matrix),
max_colour_percentile_for_diff)
this_cnorm_object_by_panel[j] = matplotlib.colors.Normalize(
vmin=-1 * this_max_value, vmax=this_max_value, clip=False)
else:
unique_field_names = numpy.unique(numpy.array(field_name_by_panel))
for this_field_name in unique_field_names:
these_panel_indices = numpy.where(
numpy.array(field_name_by_panel) == this_field_name
)[0]
this_diff_matrix = (
optimized_radar_matrix[i, ..., these_panel_indices] -
input_radar_matrix[i, ..., these_panel_indices]
)
this_max_value = numpy.percentile(
numpy.absolute(this_diff_matrix),
max_colour_percentile_for_diff)
for this_index in these_panel_indices:
this_cnorm_object_by_panel[this_index] = (
matplotlib.colors.Normalize(
vmin=-1 * this_max_value, vmax=this_max_value,
clip=False)
)
this_diff_matrix = (
optimized_radar_matrix[i, ...] - input_radar_matrix[i, ...]
)
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(this_diff_matrix, axis=0),
field_name_by_panel=field_name_by_panel,
num_panel_rows=num_panel_rows, panel_names=panel_names,
colour_map_object_by_panel=this_cmap_object_by_panel,
colour_norm_object_by_panel=this_cnorm_object_by_panel,
plot_colour_bar_by_panel=plot_colour_bar_by_panel,
font_size=FONT_SIZE_WITH_COLOUR_BARS, row_major=False)
this_title_string = '{0:s} (after minus before optimization)'.format(
this_base_title_string)
this_file_name = '{0:s}/{1:s}_optimization-diff_radar.jpg'.format(
difference_dir_name, this_base_pathless_file_name)
pyplot.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
print 'Saving figure to: "{0:s}"...'.format(this_file_name)
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
def _plot_2d_radar_cams(
radar_matrix, model_metadata_dict, cam_colour_map_object,
max_colour_prctile_for_cam, output_dir_name,
class_activation_matrix=None, ggradcam_output_matrix=None,
storm_ids=None, storm_times_unix_sec=None):
"""Plots class-activation maps for 2-D radar data.
E = number of examples
M = number of rows in spatial grid
N = number of columns in spatial grid
C = number of channels (field/height pairs)
This method will plot either `class_activation_matrix` or
`ggradcam_output_matrix`, not both.
If `storm_ids is None` and `storm_times_unix_sec is None`, will assume that
the input matrices contain probability-matched means.
:param radar_matrix: E-by-M-by-N-by-C numpy array of radar values.
:param model_metadata_dict: See doc for `_plot_3d_radar_cams`.
:param cam_colour_map_object: Same.
:param max_colour_prctile_for_cam: Same.
:param output_dir_name: Same.
:param class_activation_matrix: E-by-M-by-N numpy array of class
activations.
:param ggradcam_output_matrix: E-by-M-by-N-by-C numpy array of output values
from guided Grad-CAM.
:param storm_ids: length-E list of storm IDs (strings).
:param storm_times_unix_sec: length-E numpy array of storm times.
"""
pmm_flag = storm_ids is None and storm_times_unix_sec is None
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if list_of_layer_operation_dicts is None:
field_name_by_panel = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
panel_names = (
radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=field_name_by_panel,
heights_m_agl=training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]
)
)
plot_colour_bar_by_panel = numpy.full(
len(panel_names), True, dtype=bool)
else:
field_name_by_panel, panel_names = (
radar_plotting.layer_ops_to_field_and_panel_names(
list_of_layer_operation_dicts)
)
plot_colour_bar_by_panel = numpy.full(
len(panel_names), False, dtype=bool)
plot_colour_bar_by_panel[2::3] = True
num_examples = radar_matrix.shape[0]
num_channels = radar_matrix.shape[-1]
num_panel_rows = int(numpy.floor(numpy.sqrt(num_channels)))
if class_activation_matrix is None:
quantity_string = 'max absolute guided Grad-CAM output'
pathless_file_name_prefix = 'guided-gradcam'
else:
quantity_string = 'max class activation'
pathless_file_name_prefix = 'gradcam'
for i in range(num_examples):
_, these_axes_objects = (
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(radar_matrix[i, ...], axis=0),
field_name_by_panel=field_name_by_panel,
panel_names=panel_names, num_panel_rows=num_panel_rows,
plot_colour_bar_by_panel=plot_colour_bar_by_panel,
font_size=FONT_SIZE_WITH_COLOUR_BARS, row_major=False)
)
if class_activation_matrix is None:
this_matrix = ggradcam_output_matrix[i, ...]
this_max_contour_level = numpy.percentile(
numpy.absolute(this_matrix), max_colour_prctile_for_cam)
if this_max_contour_level == 0:
this_max_contour_level = 10.
saliency_plotting.plot_many_2d_grids_with_contours(
saliency_matrix_3d=numpy.flip(this_matrix, axis=0),
axes_objects_2d_list=these_axes_objects,
colour_map_object=cam_colour_map_object,
max_absolute_contour_level=this_max_contour_level,
contour_interval=this_max_contour_level / 10, row_major=False)
else:
this_matrix = numpy.expand_dims(
class_activation_matrix[i, ...], axis=-1)
this_matrix = numpy.repeat(
this_matrix, repeats=num_channels, axis=-1)
this_max_contour_level = numpy.percentile(
this_matrix, max_colour_prctile_for_cam)
if this_max_contour_level == 0:
this_max_contour_level = 10.
cam_plotting.plot_many_2d_grids(
class_activation_matrix_3d=numpy.flip(this_matrix, axis=0),
axes_objects_2d_list=these_axes_objects,
colour_map_object=cam_colour_map_object,
max_contour_level=this_max_contour_level,
contour_interval=this_max_contour_level / NUM_CONTOURS,
row_major=False)
if pmm_flag:
this_title_string = 'Probability-matched mean'
this_figure_file_name = '{0:s}/{1:s}_pmm_radar.jpg'.format(
output_dir_name, pathless_file_name_prefix)
else:
this_storm_time_string = time_conversion.unix_sec_to_string(
storm_times_unix_sec[i], TIME_FORMAT)
this_title_string = 'Storm "{0:s}" at {1:s}'.format(
storm_ids[i], this_storm_time_string)
this_figure_file_name = '{0:s}/{1:s}_{2:s}_{3:s}_radar.jpg'.format(
output_dir_name, pathless_file_name_prefix,
storm_ids[i].replace('_', '-'), this_storm_time_string)
this_title_string += ' ({0:s} = {1:.3f})'.format(
quantity_string, this_max_contour_level)
pyplot.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
print 'Saving figure to file: "{0:s}"...'.format(this_figure_file_name)
pyplot.savefig(this_figure_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
def _create_predictor_names(model_metadata_dict, list_of_predictor_matrices):
"""Creates predictor names for input to `permutation.run_permutation_test`.
T = number of predictor matrices
:param model_metadata_dict: See doc for `cnn.read_model_metadata`.
:param list_of_predictor_matrices: length-T list of predictor matrices,
where each item is a numpy array.
:return: predictor_names_by_matrix: length-T list, where the [q]th element
is a 1-D list of predictor names correspond to the [q]th predictor
matrix.
"""
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
if model_metadata_dict[cnn.CONV_2D3D_KEY]:
num_radar_dimensions = -1
else:
num_radar_dimensions = len(list_of_predictor_matrices[0].shape) - 2
if num_radar_dimensions == -1:
nice_refl_names = (
radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=[radar_utils.REFL_NAME],
heights_m_agl=numpy.full(1, 1000),
include_units=False))
nice_refl_names = [n[:n.find('\n')] for n in nice_refl_names]
basic_az_shear_names = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
nice_az_shear_names = (
radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=basic_az_shear_names,
heights_m_agl=numpy.full(len(basic_az_shear_names), 1000),
include_units=False))
predictor_names_by_matrix = [nice_refl_names, nice_az_shear_names]
elif num_radar_dimensions == 3:
basic_radar_names = training_option_dict[trainval_io.RADAR_FIELDS_KEY]
nice_radar_names = (
radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=basic_radar_names,
heights_m_agl=numpy.full(len(basic_radar_names), 1000),
include_units=False))
nice_radar_names = [n[:n.find('\n')] for n in nice_radar_names]
predictor_names_by_matrix = [nice_radar_names]
else:
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if list_of_layer_operation_dicts is None:
nice_radar_names = (
radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=training_option_dict[
trainval_io.RADAR_FIELDS_KEY],
heights_m_agl=training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY],
include_units=False))
nice_radar_names = [n[:n.find('\n')] for n in nice_radar_names]
else:
_, nice_radar_names = (
radar_plotting.layer_ops_to_field_and_panel_names(
list_of_layer_operation_dicts=list_of_layer_operation_dicts,
include_units=False))
nice_radar_names = [
n.replace('\n', ', ') for n in nice_radar_names
]
predictor_names_by_matrix = [nice_radar_names]
if training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is None:
return predictor_names_by_matrix
basic_sounding_names = training_option_dict[
trainval_io.SOUNDING_FIELDS_KEY]
nice_sounding_names = [
soundings.field_name_to_verbose(field_name=n, include_units=False)
for n in basic_sounding_names
]
predictor_names_by_matrix.append(nice_sounding_names)
return predictor_names_by_matrix
def _plot_2d_radar_scan(list_of_predictor_matrices,
model_metadata_dict,
allow_whitespace,
title_string=None):
"""Plots 2-D radar scan for one example.
J = number of panel rows in image
K = number of panel columns in image
:param list_of_predictor_matrices: See doc for `_plot_3d_radar_scan`.
:param model_metadata_dict: Same.
:param allow_whitespace: Same.
:param title_string: Same.
:return: figure_objects: length-1 list of figure handles (instances of
`matplotlib.figure.Figure`).
:return: axes_object_matrices: length-1 list. Each element is a J-by-K
numpy array of axes handles (instances of
`matplotlib.axes._subplots.AxesSubplot`).
"""
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if list_of_layer_operation_dicts is None:
field_name_by_panel = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
num_panels = len(field_name_by_panel)
panel_names = radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=field_name_by_panel,
heights_m_agl=training_option_dict[trainval_io.RADAR_HEIGHTS_KEY])
plot_cbar_by_panel = numpy.full(num_panels, True, dtype=bool)
else:
list_of_layer_operation_dicts = [
list_of_layer_operation_dicts[k] for k in LAYER_OP_INDICES_TO_KEEP
]
list_of_predictor_matrices[0] = list_of_predictor_matrices[0][
..., LAYER_OP_INDICES_TO_KEEP]
field_name_by_panel, panel_names = (
radar_plotting.layer_ops_to_field_and_panel_names(
list_of_layer_operation_dicts=list_of_layer_operation_dicts))
num_panels = len(field_name_by_panel)
plot_cbar_by_panel = numpy.full(num_panels, True, dtype=bool)
# if allow_whitespace:
# if len(field_name_by_panel) == 12:
# plot_cbar_by_panel[2::3] = True
# else:
# plot_cbar_by_panel[:] = True
num_panel_rows = int(numpy.floor(numpy.sqrt(num_panels)))
num_panel_columns = int(numpy.ceil(float(num_panels) / num_panel_rows))
if allow_whitespace:
figure_object = None
axes_object_matrix = None
else:
figure_object, axes_object_matrix = (
plotting_utils.create_paneled_figure(num_rows=num_panel_rows,
num_columns=num_panel_columns,
horizontal_spacing=0.,
vertical_spacing=0.,
shared_x_axis=False,
shared_y_axis=False,
keep_aspect_ratio=True))
figure_object, axes_object_matrix = (
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(list_of_predictor_matrices[0], axis=0),
field_name_by_panel=field_name_by_panel,
panel_names=panel_names,
num_panel_rows=num_panel_rows,
figure_object=figure_object,
axes_object_matrix=axes_object_matrix,
plot_colour_bar_by_panel=plot_cbar_by_panel,
font_size=FONT_SIZE_WITH_COLOUR_BARS,
row_major=False))
if allow_whitespace and title_string is not None:
pyplot.suptitle(title_string, fontsize=TITLE_FONT_SIZE)
return [figure_object], [axes_object_matrix]
def plot_examples(list_of_predictor_matrices,
storm_ids,
storm_times_unix_sec,
model_metadata_dict,
output_dir_name,
storm_activations=None):
"""Plots one or more learning examples.
E = number of examples (storm objects)
:param list_of_predictor_matrices: List created by
`testing_io.read_specific_examples`. Contains data to be plotted.
:param storm_ids: length-E list of storm IDs.
:param storm_times_unix_sec: length-E numpy array of storm times.
:param model_metadata_dict: See doc for `cnn.read_model_metadata`.
:param output_dir_name: Name of output directory (figures will be saved
here).
:param storm_activations: length-E numpy array of storm activations (may be
None). Will be included in title of each figure.
"""
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
sounding_field_names = training_option_dict[
trainval_io.SOUNDING_FIELDS_KEY]
plot_soundings = sounding_field_names is not None
if plot_soundings:
list_of_metpy_dictionaries = dl_utils.soundings_to_metpy_dictionaries(
sounding_matrix=list_of_predictor_matrices[-1],
field_names=sounding_field_names)
else:
list_of_metpy_dictionaries = None
num_radar_dimensions = len(list_of_predictor_matrices[0].shape) - 2
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if num_radar_dimensions == 2:
if list_of_layer_operation_dicts is None:
field_name_by_panel = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
panel_names = (
radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=field_name_by_panel,
heights_m_agl=training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]))
plot_colour_bar_by_panel = numpy.full(len(panel_names),
True,
dtype=bool)
else:
field_name_by_panel, panel_names = (
radar_plotting.layer_ops_to_field_and_panel_names(
list_of_layer_operation_dicts))
plot_colour_bar_by_panel = numpy.full(len(panel_names),
False,
dtype=bool)
plot_colour_bar_by_panel[2::3] = True
else:
field_name_by_panel = None
panel_names = None
plot_colour_bar_by_panel = None
az_shear_field_names = training_option_dict[trainval_io.RADAR_FIELDS_KEY]
num_az_shear_fields = len(az_shear_field_names)
num_storms = len(storm_ids)
myrorss_2d3d = len(list_of_predictor_matrices) == 3
for i in range(num_storms):
this_time_string = time_conversion.unix_sec_to_string(
storm_times_unix_sec[i], TIME_FORMAT)
this_base_title_string = 'Storm "{0:s}" at {1:s}'.format(
storm_ids[i], this_time_string)
if storm_activations is not None:
this_base_title_string += ' (activation = {0:.3f})'.format(
storm_activations[i])
this_base_file_name = '{0:s}/storm={1:s}_{2:s}'.format(
output_dir_name, storm_ids[i].replace('_', '-'), this_time_string)
if plot_soundings:
sounding_plotting.plot_sounding(
sounding_dict_for_metpy=list_of_metpy_dictionaries[i],
title_string=this_base_title_string)
this_file_name = '{0:s}_sounding.jpg'.format(this_base_file_name)
print 'Saving figure to: "{0:s}"...'.format(this_file_name)
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
if myrorss_2d3d:
this_reflectivity_matrix_dbz = numpy.flip(
list_of_predictor_matrices[0][i, ..., 0], axis=0)
this_num_heights = this_reflectivity_matrix_dbz.shape[-1]
this_num_panel_rows = int(numpy.floor(
numpy.sqrt(this_num_heights)))
_, these_axes_objects = radar_plotting.plot_3d_grid_without_coords(
field_matrix=this_reflectivity_matrix_dbz,
field_name=radar_utils.REFL_NAME,
grid_point_heights_metres=training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY],
ground_relative=True,
num_panel_rows=this_num_panel_rows,
font_size=FONT_SIZE_SANS_COLOUR_BARS)
this_colour_map_object, this_colour_norm_object = (
radar_plotting.get_default_colour_scheme(
radar_utils.REFL_NAME))
plotting_utils.add_colour_bar(
axes_object_or_list=these_axes_objects,
values_to_colour=this_reflectivity_matrix_dbz,
colour_map=this_colour_map_object,
colour_norm_object=this_colour_norm_object,
orientation='horizontal',
extend_min=True,
extend_max=True)
this_title_string = '{0:s}; {1:s}'.format(this_base_title_string,
radar_utils.REFL_NAME)
this_file_name = '{0:s}_reflectivity.jpg'.format(
this_base_file_name)
pyplot.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
print 'Saving figure to: "{0:s}"...'.format(this_file_name)
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
this_az_shear_matrix_s01 = numpy.flip(
list_of_predictor_matrices[1][i, ..., 0], axis=0)
_, these_axes_objects = (
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=this_az_shear_matrix_s01,
field_name_by_panel=az_shear_field_names,
panel_names=az_shear_field_names,
num_panel_rows=1,
plot_colour_bar_by_panel=numpy.full(num_az_shear_fields,
False,
dtype=bool),
font_size=FONT_SIZE_SANS_COLOUR_BARS))
this_colour_map_object, this_colour_norm_object = (
radar_plotting.get_default_colour_scheme(
radar_utils.LOW_LEVEL_SHEAR_NAME))
plotting_utils.add_colour_bar(
axes_object_or_list=these_axes_objects,
values_to_colour=this_az_shear_matrix_s01,
colour_map=this_colour_map_object,
colour_norm_object=this_colour_norm_object,
orientation='horizontal',
extend_min=True,
extend_max=True)
this_file_name = '{0:s}_shear.jpg'.format(this_base_file_name)
pyplot.suptitle(this_base_title_string, fontsize=TITLE_FONT_SIZE)
print 'Saving figure to: "{0:s}"...'.format(this_file_name)
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
continue
this_radar_matrix = list_of_predictor_matrices[0]
if num_radar_dimensions == 2:
this_num_channels = this_radar_matrix.shape[-1]
this_num_panel_rows = int(
numpy.floor(numpy.sqrt(this_num_channels)))
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(this_radar_matrix[i, ...], axis=0),
field_name_by_panel=field_name_by_panel,
panel_names=panel_names,
num_panel_rows=this_num_panel_rows,
plot_colour_bar_by_panel=plot_colour_bar_by_panel,
font_size=FONT_SIZE_WITH_COLOUR_BARS,
row_major=False)
this_title_string = this_base_title_string + ''
pyplot.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
this_file_name = '{0:s}.jpg'.format(this_base_file_name)
print 'Saving figure to: "{0:s}"...'.format(this_file_name)
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
continue
radar_field_names = training_option_dict[trainval_io.RADAR_FIELDS_KEY]
radar_heights_m_agl = training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]
for j in range(len(radar_field_names)):
this_num_heights = this_radar_matrix.shape[-2]
this_num_panel_rows = int(numpy.floor(
numpy.sqrt(this_num_heights)))
_, these_axes_objects = radar_plotting.plot_3d_grid_without_coords(
field_matrix=numpy.flip(this_radar_matrix[i, ..., j], axis=0),
field_name=radar_field_names[j],
grid_point_heights_metres=radar_heights_m_agl,
ground_relative=True,
num_panel_rows=this_num_panel_rows,
font_size=FONT_SIZE_SANS_COLOUR_BARS)
this_colour_map_object, this_colour_norm_object = (
radar_plotting.get_default_colour_scheme(radar_field_names[j]))
plotting_utils.add_colour_bar(
axes_object_or_list=these_axes_objects,
values_to_colour=this_radar_matrix[i, ..., j],
colour_map=this_colour_map_object,
colour_norm_object=this_colour_norm_object,
orientation='horizontal',
extend_min=True,
extend_max=True)
this_title_string = '{0:s}; {1:s}'.format(this_base_title_string,
radar_field_names[j])
this_file_name = '{0:s}_{1:s}.jpg'.format(
this_base_file_name, radar_field_names[j].replace('_', '-'))
pyplot.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
print 'Saving figure to: "{0:s}"...'.format(this_file_name)
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
def _plot_2d_radar_difference(difference_matrix,
colour_map_object,
max_colour_percentile,
model_metadata_dict,
backwards_opt_dict,
output_dir_name,
example_index=None,
significance_matrix=None):
"""Plots difference (after minus before optimization) for 2-D radar data.
M = number of rows in spatial grid
N = number of columns in spatial grid
C = number of channels
:param difference_matrix: M-by-N-by-C numpy array of differences (after
minus before optimization).
:param colour_map_object: See doc for `_plot_3d_radar_difference`.
:param max_colour_percentile: Same.
:param model_metadata_dict: Same.
:param backwards_opt_dict: Same.
:param output_dir_name: Same.
:param example_index: Same.
:param significance_matrix: M-by-N-by-C numpy array of Boolean flags,
indicating where these differences are significantly different than
differences from another backwards optimization.
"""
pmm_flag = backwards_opt.MEAN_FINAL_ACTIVATION_KEY in backwards_opt_dict
if pmm_flag:
initial_activation = backwards_opt_dict[
backwards_opt.MEAN_INITIAL_ACTIVATION_KEY]
final_activation = backwards_opt_dict[
backwards_opt.MEAN_FINAL_ACTIVATION_KEY]
full_storm_id_string = None
storm_time_string = None
else:
initial_activation = backwards_opt_dict[
backwards_opt.INITIAL_ACTIVATIONS_KEY][example_index]
final_activation = backwards_opt_dict[
backwards_opt.FINAL_ACTIVATIONS_KEY][example_index]
full_storm_id_string = backwards_opt_dict[
backwards_opt.FULL_IDS_KEY][example_index]
storm_time_string = time_conversion.unix_sec_to_string(
backwards_opt_dict[backwards_opt.STORM_TIMES_KEY][example_index],
plot_input_examples.TIME_FORMAT)
conv_2d3d = model_metadata_dict[cnn.CONV_2D3D_KEY]
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
if conv_2d3d:
num_fields = len(training_option_dict[trainval_io.RADAR_FIELDS_KEY])
radar_heights_m_agl = numpy.full(num_fields,
radar_utils.SHEAR_HEIGHT_M_ASL,
dtype=int)
else:
radar_heights_m_agl = training_option_dict[
trainval_io.RADAR_HEIGHTS_KEY]
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if list_of_layer_operation_dicts is None:
field_name_by_panel = training_option_dict[
trainval_io.RADAR_FIELDS_KEY]
panel_names = radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=field_name_by_panel, heights_m_agl=radar_heights_m_agl)
else:
field_name_by_panel, panel_names = (
radar_plotting.layer_ops_to_field_and_panel_names(
list_of_layer_operation_dicts=list_of_layer_operation_dicts))
num_panels = len(field_name_by_panel)
plot_cbar_by_panel = numpy.full(num_panels, True, dtype=bool)
cmap_object_by_panel = [colour_map_object] * num_panels
cnorm_object_by_panel = [None] * num_panels
for j in range(num_panels):
this_max_colour_value = numpy.percentile(
numpy.absolute(difference_matrix[..., j]), max_colour_percentile)
cnorm_object_by_panel[j] = matplotlib.colors.Normalize(
vmin=-1 * this_max_colour_value,
vmax=this_max_colour_value,
clip=False)
num_panel_rows = int(numpy.floor(numpy.sqrt(num_panels)))
figure_object, axes_object_matrix = (
radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(difference_matrix, axis=0),
field_name_by_panel=field_name_by_panel,
num_panel_rows=num_panel_rows,
panel_names=panel_names,
row_major=False,
colour_map_object_by_panel=cmap_object_by_panel,
colour_norm_object_by_panel=cnorm_object_by_panel,
plot_colour_bar_by_panel=plot_cbar_by_panel,
font_size=FONT_SIZE_WITH_COLOUR_BARS))
if significance_matrix is not None:
significance_plotting.plot_many_2d_grids_without_coords(
significance_matrix=numpy.flip(significance_matrix, axis=0),
axes_object_matrix=axes_object_matrix,
row_major=False)
if pmm_flag:
this_title_string = 'PMM'
else:
this_title_string = 'Storm "{0:s}" at {1:s}'.format(
full_storm_id_string, storm_time_string)
this_title_string += '; activation from {0:.2e} to {1:.2e}'.format(
initial_activation, final_activation)
figure_object.suptitle(this_title_string, fontsize=TITLE_FONT_SIZE)
output_file_name = plot_input_examples.metadata_to_radar_fig_file_name(
output_dir_name=output_dir_name,
pmm_flag=pmm_flag,
full_storm_id_string=full_storm_id_string,
storm_time_string=storm_time_string,
radar_field_name='shear' if conv_2d3d else None)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
figure_object.savefig(output_file_name,
dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0,
bbox_inches='tight')
pyplot.close(figure_object)
def _plot_comparison(
predictor_matrix, model_metadata_dict, machine_mask_matrix_3d,
human_mask_matrix_3d, iou_by_channel, positive_flag, output_file_name):
"""Plots comparison between human and machine interpretation maps.
M = number of rows in grid (physical space)
N = number of columns in grid (physical space)
C = number of channels
:param predictor_matrix: M-by-N-by-C numpy array of predictors.
:param model_metadata_dict: Dictionary returned by
`cnn.read_model_metadata`.
:param machine_mask_matrix_3d: M-by-N-by-C numpy array of Boolean flags,
indicating where machine interpretation value is strongly positive or
negative.
:param human_mask_matrix_3d: Same.
:param iou_by_channel: length-C numpy array of IoU values (intersection over
union) between human and machine masks.
:param positive_flag: Boolean flag. If True (False), masks indicate where
interpretation value is strongly positive (negative).
:param output_file_name: Path to output file (figure will be saved here).
"""
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
list_of_layer_operation_dicts = model_metadata_dict[
cnn.LAYER_OPERATIONS_KEY]
if list_of_layer_operation_dicts is None:
field_name_by_panel = training_option_dict[trainval_io.RADAR_FIELDS_KEY]
panel_names = radar_plotting.radar_fields_and_heights_to_panel_names(
field_names=field_name_by_panel,
heights_m_agl=training_option_dict[trainval_io.RADAR_HEIGHTS_KEY]
)
plot_colour_bar_by_panel = numpy.full(
len(field_name_by_panel), True, dtype=bool
)
else:
field_name_by_panel, panel_names = (
radar_plotting.layer_ops_to_field_and_panel_names(
list_of_layer_operation_dicts=list_of_layer_operation_dicts
)
)
plot_colour_bar_by_panel = numpy.full(
len(field_name_by_panel), False, dtype=bool
)
plot_colour_bar_by_panel[2::3] = True
num_panels = len(field_name_by_panel)
num_panel_rows = int(numpy.floor(
numpy.sqrt(num_panels)
))
for k in range(num_panels):
panel_names[k] += '\n{0:s} IoU = {1:.3f}'.format(
'Positive' if positive_flag else 'Negative',
iou_by_channel[k]
)
_, axes_object_matrix = radar_plotting.plot_many_2d_grids_without_coords(
field_matrix=numpy.flip(predictor_matrix, axis=0),
field_name_by_panel=field_name_by_panel, panel_names=panel_names,
num_panel_rows=num_panel_rows,
plot_colour_bar_by_panel=plot_colour_bar_by_panel, font_size=14,
row_major=False)
for k in range(num_panels):
_plot_comparison_one_channel(
human_mask_matrix_3d=human_mask_matrix_3d,
machine_mask_matrix_3d=numpy.flip(machine_mask_matrix_3d, axis=0),
channel_index=k, axes_object_matrix=axes_object_matrix)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0., bbox_inches='tight')
pyplot.close()
