def _check_input_events(event_x_coords_metres, event_y_coords_metres,
integer_event_ids):
"""Checks inputs to `count_events_on_equidistant_grid`.
:param event_x_coords_metres: See doc for
`count_events_on_equidistant_grid`.
:param event_y_coords_metres: Same.
:param integer_event_ids: Same.
"""
error_checking.assert_is_numpy_array_without_nan(event_x_coords_metres)
error_checking.assert_is_numpy_array(event_x_coords_metres,
num_dimensions=1)
num_events = len(event_x_coords_metres)
error_checking.assert_is_numpy_array_without_nan(event_y_coords_metres)
error_checking.assert_is_numpy_array(event_y_coords_metres,
exact_dimensions=numpy.array(
[num_events]))
if integer_event_ids is not None:
error_checking.assert_is_integer_numpy_array(integer_event_ids)
error_checking.assert_is_numpy_array(integer_event_ids,
exact_dimensions=numpy.array(
[num_events]))
def plot_error_distribution(error_values, min_error_to_plot, max_error_to_plot,
axes_object):
"""Plots one error distribution.
:param error_values: 1-D numpy array of signed errors.
:param min_error_to_plot: See doc for `plot_error_dist_many_heights`.
:param max_error_to_plot: Same.
:param axes_object: Same.
"""
error_checking.assert_is_numpy_array_without_nan(error_values)
error_checking.assert_is_numpy_array(error_values, num_dimensions=1)
error_checking.assert_is_greater(max_error_to_plot, min_error_to_plot)
boxplot_style_dict = {'color': 'k', 'linewidth': 2}
axes_object.boxplot(error_values,
vert=True,
notch=False,
sym='o',
whis=(5, 95),
medianprops=boxplot_style_dict,
boxprops=boxplot_style_dict,
whiskerprops=boxplot_style_dict,
capprops=boxplot_style_dict)
axes_object.set_ylim(min_error_to_plot, max_error_to_plot)
axes_object.set_xticks([], [])
def xy_to_scalar_displacements_and_bearings(
x_displacements_metres, y_displacements_metres):
"""For each displacement vector, converts x-y to magnitude and direction.
:param x_displacements_metres: numpy array of eastward displacements.
:param y_displacements_metres: equivalent-size numpy array of northward
displacements.
:return: scalar_displacements_metres: equivalent-size numpy array of total
displacements.
:return: geodetic_bearings_deg: equivalent-size numpy array of geodetic
bearings (from start point to end point, measured clockwise from due
north).
"""
error_checking.assert_is_numpy_array_without_nan(x_displacements_metres)
error_checking.assert_is_numpy_array_without_nan(y_displacements_metres)
error_checking.assert_is_numpy_array(
y_displacements_metres,
exact_dimensions=numpy.array(y_displacements_metres.shape))
scalar_displacements_metres = numpy.sqrt(
x_displacements_metres ** 2 + y_displacements_metres ** 2)
standard_bearings_deg = RADIANS_TO_DEGREES * numpy.arctan2(
y_displacements_metres, x_displacements_metres)
return scalar_displacements_metres, standard_to_geodetic_angles(
standard_bearings_deg)
def update_z_score_params(z_score_param_dict, new_data_matrix):
"""Uses new data to update parameters for z-score normalization.
:param z_score_param_dict: Dictionary with the following keys.
z_score_param_dict['num_values']: Number of values on which current
estimates are based.
z_score_param_dict['mean_value']: Current estimate of mean.
z_score_param_dict['mean_of_squares']: Current estimate of mean of squares.
:param new_data_matrix: numpy array with new data.
:return: z_score_param_dict: Same as input but with updated values.
"""
error_checking.assert_is_numpy_array_without_nan(new_data_matrix)
these_means = numpy.array(
[z_score_param_dict[MEAN_VALUE_KEY],
numpy.mean(new_data_matrix)])
these_weights = numpy.array(
[z_score_param_dict[NUM_VALUES_KEY], new_data_matrix.size])
z_score_param_dict[MEAN_VALUE_KEY] = numpy.average(these_means,
weights=these_weights)
these_means = numpy.array([
z_score_param_dict[MEAN_OF_SQUARES_KEY],
numpy.mean(new_data_matrix**2)
])
these_weights = numpy.array(
[z_score_param_dict[NUM_VALUES_KEY], new_data_matrix.size])
z_score_param_dict[MEAN_OF_SQUARES_KEY] = numpy.average(
these_means, weights=these_weights)
z_score_param_dict[NUM_VALUES_KEY] += new_data_matrix.size
return z_score_param_dict
def plot_predictor_2d(predictor_matrix,
colour_map_object,
colour_norm_object=None,
min_colour_value=None,
max_colour_value=None,
axes_object=None):
"""Plots predictor variable on 2-D grid.
If `colour_norm_object is None`, both `min_colour_value` and
`max_colour_value` must be specified.
M = number of rows in grid
N = number of columns in grid
:param predictor_matrix: M-by-N numpy array of predictor values.
:param colour_map_object: Instance of `matplotlib.pyplot.cm`.
:param min_colour_value: Minimum value in colour scheme.
:param max_colour_value: Max value in colour scheme.
:param axes_object: Instance of `matplotlib.axes._subplots.AxesSubplot`.
Will plot on these axes.
:return: colour_bar_object: Colour bar (instance of
`matplotlib.pyplot.colorbar`) created by this method.
"""
error_checking.assert_is_numpy_array_without_nan(predictor_matrix)
error_checking.assert_is_numpy_array(predictor_matrix, num_dimensions=2)
if axes_object is None:
_, axes_object = pyplot.subplots(1,
1,
figsize=(FIGURE_WIDTH_INCHES,
FIGURE_HEIGHT_INCHES))
if colour_norm_object is None:
error_checking.assert_is_greater(max_colour_value, min_colour_value)
colour_norm_object = None
else:
if hasattr(colour_norm_object, 'boundaries'):
min_colour_value = colour_norm_object.boundaries[0]
max_colour_value = colour_norm_object.boundaries[-1]
else:
min_colour_value = colour_norm_object.vmin
max_colour_value = colour_norm_object.vmax
axes_object.pcolormesh(predictor_matrix,
cmap=colour_map_object,
norm=colour_norm_object,
vmin=min_colour_value,
vmax=max_colour_value,
shading='flat',
edgecolors='None')
axes_object.set_xticks([])
axes_object.set_yticks([])
return plotting_utils.add_colour_bar(axes_object_or_list=axes_object,
values_to_colour=predictor_matrix,
colour_map=colour_map_object,
colour_norm_object=colour_norm_object,
orientation='vertical')
def _check_args_one_step(predictor_matrix, permuted_flag_matrix,
scalar_channel_flags, shuffle_profiles_together,
num_bootstrap_reps):
"""Checks input args for `run_*_test_one_step`.
:param predictor_matrix: See doc for `run_forward_test_one_step` or
`run_backwards_test_one_step`.
:param permuted_flag_matrix: Same.
:param scalar_channel_flags: Same.
:param shuffle_profiles_together: Same.
:param num_bootstrap_reps: Same.
:return: num_bootstrap_reps: Same as input but maxxed with 1.
"""
error_checking.assert_is_numpy_array_without_nan(predictor_matrix)
num_predictor_dim = len(predictor_matrix.shape)
error_checking.assert_is_geq(num_predictor_dim, 3)
error_checking.assert_is_leq(num_predictor_dim, 3)
error_checking.assert_is_boolean_numpy_array(permuted_flag_matrix)
these_expected_dim = numpy.array(predictor_matrix.shape[1:], dtype=int)
error_checking.assert_is_numpy_array(permuted_flag_matrix,
exact_dimensions=these_expected_dim)
error_checking.assert_is_boolean_numpy_array(scalar_channel_flags)
these_expected_dim = numpy.array([predictor_matrix.shape[-1]], dtype=int)
error_checking.assert_is_numpy_array(scalar_channel_flags,
exact_dimensions=these_expected_dim)
error_checking.assert_is_boolean(shuffle_profiles_together)
error_checking.assert_is_integer(num_bootstrap_reps)
return numpy.maximum(num_bootstrap_reps, 1)
def gaussian_smooth_2d_field(field_matrix, standard_deviation_pixels,
cutoff_radius_pixels):
"""Applies Gaussian smoother to 2-D field.
M = number of rows in grid
N = number of columns in grid
:param field_matrix: M-by-N numpy array with values in field.
:param standard_deviation_pixels: Standard deviation of Gaussian kernel
(pixels).
:param cutoff_radius_pixels: Cutoff radius of Gaussian kernel (pixels).
:return: field_matrix: Smoothed version of input.
"""
error_checking.assert_is_numpy_array_without_nan(field_matrix)
error_checking.assert_is_numpy_array(field_matrix, num_dimensions=2)
error_checking.assert_is_greater(standard_deviation_pixels, 0.)
error_checking.assert_is_greater(cutoff_radius_pixels,
standard_deviation_pixels)
return gaussian_filter(input=field_matrix,
sigma=standard_deviation_pixels,
order=0,
mode='reflect',
truncate=cutoff_radius_pixels)
def classify_values(input_values, class_cutoffs, non_negative_only=True):
"""Assigns each element of input array to one class.
N = number of values to classify
C = number of classes
c = C - 1 = number of cutoffs
:param input_values: length-N numpy array of values to classify.
:param class_cutoffs: length-c numpy array of class cutoffs.
:param non_negative_only: Boolean flag. If True, all values (class ranges
and values to classify) must be non-negative.
:return: class_labels: length-N numpy array of integer class labels.
"""
_, class_minima, class_maxima = classification_cutoffs_to_ranges(
class_cutoffs, non_negative_only=non_negative_only)
error_checking.assert_is_numpy_array_without_nan(input_values)
error_checking.assert_is_numpy_array(input_values, num_dimensions=1)
if non_negative_only:
error_checking.assert_is_geq_numpy_array(input_values, 0.)
num_inputs = len(input_values)
class_labels = numpy.full(num_inputs, -1, dtype=int)
num_classes = len(class_minima)
for k in range(num_classes):
these_flags = numpy.logical_and(
input_values >= class_minima[k], input_values < class_maxima[k])
these_indices = numpy.where(these_flags)[0]
class_labels[these_indices] = k
return class_labels
def get_locating_variable(tfp_matrix_kelvins_m02,
projected_velocity_matrix_m_s01):
"""Computes locating variable at each grid point.
The "locating variable" is the product of the absolute TFP (thermal front
parameter) and projected wind velocity (in the direction of the thermal
gradient). Large positive values indicate the presence of a cold front,
while large negative values indicate the presence of a warm front.
M = number of rows in grid
N = number of columns in grid
:param tfp_matrix_kelvins_m02: M-by-N numpy array created by
`get_thermal_front_param`.
:param projected_velocity_matrix_m_s01: M-by-N numpy array created by
`project_wind_to_thermal_gradient`.
:return: locating_var_matrix_m01_s01: M-by-N numpy array with locating
variable (units of m^-1 s^-1) at each grid point.
"""
error_checking.assert_is_numpy_array_without_nan(tfp_matrix_kelvins_m02)
error_checking.assert_is_numpy_array(tfp_matrix_kelvins_m02,
num_dimensions=2)
error_checking.assert_is_numpy_array_without_nan(
projected_velocity_matrix_m_s01)
error_checking.assert_is_numpy_array(projected_velocity_matrix_m_s01,
exact_dimensions=numpy.array(
tfp_matrix_kelvins_m02.shape))
return (numpy.absolute(tfp_matrix_kelvins_m02) *
projected_velocity_matrix_m_s01)
def xy_components_to_displacements_and_bearings(x_displacements_metres,
y_displacements_metres):
"""For each pair of x- and y-displacement, gets total dsplcmnt and bearing.
P = number of points
:param x_displacements_metres: length-P numpy array of eastward
displacements.
:param y_displacements_metres: length-P numpy array of northward
displacements.
:return: scalar_displacements_metres: length-P numpy array of total
displacements.
:return: geodetic_bearings_deg: length-P numpy array of geodetic bearings
(measured clockwise from due north).
"""
error_checking.assert_is_numpy_array_without_nan(x_displacements_metres)
error_checking.assert_is_numpy_array(
x_displacements_metres, num_dimensions=1)
num_points = len(x_displacements_metres)
error_checking.assert_is_numpy_array_without_nan(y_displacements_metres)
error_checking.assert_is_numpy_array(
y_displacements_metres, exact_dimensions=numpy.array([num_points]))
scalar_displacements_metres = numpy.sqrt(
x_displacements_metres ** 2 + y_displacements_metres ** 2)
standard_bearings_deg = RADIANS_TO_DEGREES * numpy.arctan2(
y_displacements_metres, x_displacements_metres)
return scalar_displacements_metres, standard_to_geodetic_angles(
standard_bearings_deg)
def create_histogram(input_values, num_bins, min_value, max_value):
"""Creates a histogram with uniform bin-spacing.
N = number of input values
K = number of bins
:param input_values: length-N numpy array of input values (to be binned).
:param num_bins: Number of bins.
:param min_value: Minimum value to include in histogram. Any input value <
`min_value` will be assigned to the first bin.
:param max_value: Maximum value to include in histogram. Any input value >
`max_value` will be assigned to the last bin.
:return: input_to_bin_indices: length-N numpy array of bin indices. If
input_values[i] = j, the [i]th input value belongs in the [j]th bin.
:return: num_examples_by_bin: length-K numpy array, where the [j]th value is
the number of inputs assigned to the [j]th bin.
"""
error_checking.assert_is_numpy_array_without_nan(input_values)
error_checking.assert_is_numpy_array(input_values, num_dimensions=1)
error_checking.assert_is_integer(num_bins)
error_checking.assert_is_geq(num_bins, 2)
error_checking.assert_is_greater(max_value, min_value)
bin_cutoffs = numpy.linspace(min_value, max_value, num=num_bins + 1)
input_to_bin_indices = numpy.digitize(
input_values, bin_cutoffs, right=False) - 1
input_to_bin_indices[input_to_bin_indices < 0] = 0
input_to_bin_indices[input_to_bin_indices > num_bins - 1] = num_bins - 1
num_examples_by_bin = numpy.full(num_bins, -1, dtype=int)
for j in range(num_bins):
num_examples_by_bin[j] = numpy.sum(input_to_bin_indices == j)
return input_to_bin_indices, num_examples_by_bin
def run_guided_gradcam(orig_model_object,
list_of_input_matrices,
target_layer_name,
class_activation_matrix,
new_model_object=None):
"""Runs guided Grad-CAM.
M = number of rows in grid
N = number of columns in grid
C = number of channels
:param orig_model_object: Original model (trained instance of
`keras.models.Model` or `keras.models.Sequential`).
:param list_of_input_matrices: See doc for `run_gradcam`.
:param target_layer_name: Same.
:param class_activation_matrix: Same.
:param new_model_object: New model (created by `_change_backprop_function`),
to be used for guided backprop.
:return: ggradcam_output_matrix: M-by-N-by-C numpy array of output values.
:return: new_model_object: See input doc.
"""
# Check input args.
error_checking.assert_is_string(target_layer_name)
error_checking.assert_is_list(list_of_input_matrices)
error_checking.assert_is_numpy_array_without_nan(class_activation_matrix)
for q in range(len(list_of_input_matrices)):
error_checking.assert_is_numpy_array(list_of_input_matrices[q])
if list_of_input_matrices[q].shape[0] != 1:
list_of_input_matrices[q] = numpy.expand_dims(
list_of_input_matrices[q], axis=0)
# Do the dirty work.
if new_model_object is None:
_register_guided_backprop()
new_model_object = _change_backprop_function(
model_object=orig_model_object)
input_index = _find_relevant_input_matrix(
list_of_input_matrices=list_of_input_matrices,
num_spatial_dim=len(class_activation_matrix.shape))
saliency_function = _make_saliency_function(model_object=new_model_object,
layer_name=target_layer_name,
input_index=input_index)
saliency_matrix = saliency_function(list_of_input_matrices + [0])[0]
print 'Minimum saliency = {0:.4e} ... max saliency = {1:.4e}'.format(
numpy.min(saliency_matrix), numpy.max(saliency_matrix))
ggradcam_output_matrix = saliency_matrix * class_activation_matrix[
..., numpy.newaxis]
ggradcam_output_matrix = ggradcam_output_matrix[0, ...]
# ggradcam_output_matrix = _normalize_guided_gradcam_output(
# ggradcam_output_matrix[0, ...])
return ggradcam_output_matrix, new_model_object
def write_file(
pickle_file_name, activation_matrix, storm_ids, storm_times_unix_sec,
model_file_name, component_type_string, target_class=None,
layer_name=None, neuron_index_matrix=None, channel_indices=None):
"""Writes activations to Pickle file.
E = number of examples (storm objects)
C = number of model components (classes, neurons, or channels) for which
activations were computed
:param pickle_file_name: Path to output file.
:param activation_matrix: E-by-C numpy array of activations, where
activation_matrix[i, j] = activation of the [j]th model component for
the [i]th example.
:param storm_ids: length-E list of storm IDs.
:param storm_times_unix_sec: length-E numpy array of storm times.
:param model_file_name: Path to file with trained model.
:param component_type_string: See doc for `check_metadata`.
:param target_class: Same.
:param layer_name: Same.
:param neuron_index_matrix: Same.
:param channel_indices: Same.
"""
num_components = check_metadata(
component_type_string=component_type_string, target_class=target_class,
layer_name=layer_name, neuron_index_matrix=neuron_index_matrix,
channel_indices=channel_indices)
error_checking.assert_is_string(model_file_name)
error_checking.assert_is_string_list(storm_ids)
error_checking.assert_is_numpy_array(
numpy.array(storm_ids), num_dimensions=1)
num_examples = len(storm_ids)
error_checking.assert_is_integer_numpy_array(storm_times_unix_sec)
error_checking.assert_is_numpy_array(
storm_times_unix_sec, exact_dimensions=numpy.array([num_examples]))
error_checking.assert_is_numpy_array_without_nan(activation_matrix)
error_checking.assert_is_numpy_array(
activation_matrix,
exact_dimensions=numpy.array([num_examples, num_components]))
metadata_dict = {
STORM_IDS_KEY: storm_ids,
STORM_TIMES_KEY: storm_times_unix_sec,
MODEL_FILE_NAME_KEY: model_file_name,
COMPONENT_TYPE_KEY: component_type_string,
TARGET_CLASS_KEY: target_class,
LAYER_NAME_KEY: layer_name,
NEURON_INDICES_KEY: neuron_index_matrix,
CHANNEL_INDICES_KEY: channel_indices,
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(activation_matrix, pickle_file_handle)
pickle.dump(metadata_dict, pickle_file_handle)
pickle_file_handle.close()
def plot_skeleton_line(skeleton_line_x_coords,
skeleton_line_y_coords,
axes_object,
line_colour=DEFAULT_SKELETON_LINE_COLOUR,
line_width=DEFAULT_LINE_WIDTH):
"""Plots skeleton line through polygon.
P = number of points in skeleton line
:param skeleton_line_x_coords: length-P numpy array with x-coordinates on
skeleton line.
:param skeleton_line_y_coords: length-P numpy array with y-coordinates on
skeleton line.
:param axes_object: Instance of `matplotlib.axes._subplots.AxesSubplot`.
:param line_colour: Colour of skeleton line (in any format accepted by
`matplotlib.colors`).
:param line_width: Width of skeleton line (real positive number).
"""
error_checking.assert_is_numpy_array_without_nan(skeleton_line_x_coords)
error_checking.assert_is_numpy_array(skeleton_line_x_coords,
num_dimensions=1)
num_points = len(skeleton_line_x_coords)
error_checking.assert_is_numpy_array_without_nan(skeleton_line_y_coords)
error_checking.assert_is_numpy_array(skeleton_line_y_coords,
exact_dimensions=numpy.array(
[num_points]))
axes_object.plot(skeleton_line_x_coords,
skeleton_line_y_coords,
color=line_colour,
linestyle='solid',
linewidth=line_width)
def plot_error_dist_many_heights(error_matrix, heights_m_agl,
min_error_to_plot, max_error_to_plot,
axes_object):
"""Plots error distribution at each height on the same axes.
E = number of examples
H = number of heights
:param error_matrix: E-by-H numpy array of signed errors.
:param heights_m_agl: length-H numpy array of heights (metres above ground
level).
:param min_error_to_plot: Lower limit for x-axis.
:param max_error_to_plot: Upper limit for x-axis.
:param axes_object: Will plot on these axes (instance of
`matplotlib.axes._subplots.AxesSubplot`).
"""
error_checking.assert_is_numpy_array_without_nan(error_matrix)
error_checking.assert_is_numpy_array(error_matrix, num_dimensions=2)
num_heights = error_matrix.shape[1]
error_checking.assert_is_geq_numpy_array(heights_m_agl, 0.)
error_checking.assert_is_numpy_array(heights_m_agl,
exact_dimensions=numpy.array(
[num_heights], dtype=int))
error_checking.assert_is_greater(max_error_to_plot, min_error_to_plot)
boxplot_style_dict = {'color': 'k', 'linewidth': 2}
y_values = numpy.linspace(0, num_heights - 1, num=num_heights, dtype=float)
for j in range(num_heights):
axes_object.boxplot(error_matrix[:, j],
widths=0.9,
vert=False,
notch=False,
sym='o',
whis=(5, 95),
medianprops=boxplot_style_dict,
boxprops=boxplot_style_dict,
whiskerprops=boxplot_style_dict,
capprops=boxplot_style_dict,
positions=y_values[[j]])
axes_object.set_xlim(min_error_to_plot, max_error_to_plot)
y_tick_strings = profile_plotting.create_height_labels(
tick_values_km_agl=METRES_TO_KM * heights_m_agl, use_log_scale=False)
for j in range(len(y_tick_strings)):
if numpy.mod(j, 5) == 0:
continue
y_tick_strings[j] = ' '
axes_object.set_yticklabels(y_tick_strings)
axes_object.set_ylabel('Height (km AGL)')
def plot_scattered_points(axes_object=None,
basemap_object=None,
latitudes_deg=None,
longitudes_deg=None,
field_values=None,
marker_size=DEFAULT_MARKER_SIZE,
marker_edge_width=DEFAULT_MARKER_EDGE_WIDTH,
colour_map=None,
colour_minimum=None,
colour_maximum=None):
"""Plots values of a single NWP field at scattered points.
These may be interpolated values.
P = number of points
:param axes_object: Instance of `matplotlib.axes._subplots.AxesSubplot`.
:param basemap_object: Instance of `mpl_toolkits.basemap.Basemap`.
:param latitudes_deg: length-P numpy array of latitudes (deg N).
:param longitudes_deg: length-P numpy array of longitudes (deg E).
:param field_values: length-P numpy array with values of field.
:param marker_size: Size of each marker (circle).
:param marker_edge_width: Line width of black edge around each marker
(circle).
:param colour_map: Instance of `matplotlib.pyplot.cm`.
:param colour_minimum: Minimum value for colour map.
:param colour_maximum: Maximum value for colour map.
"""
error_checking.assert_is_numpy_array_without_nan(field_values)
error_checking.assert_is_numpy_array(field_values, num_dimensions=1)
num_points = len(field_values)
error_checking.assert_is_valid_lat_numpy_array(latitudes_deg)
error_checking.assert_is_numpy_array(latitudes_deg,
exact_dimensions=numpy.array(
[num_points]))
longitudes_deg = lng_conversion.convert_lng_positive_in_west(
longitudes_deg, allow_nan=False)
error_checking.assert_is_numpy_array(longitudes_deg,
exact_dimensions=numpy.array(
[num_points]))
error_checking.assert_is_greater(colour_maximum, colour_minimum)
x_coords_metres, y_coords_metres = basemap_object(longitudes_deg,
latitudes_deg)
axes_object.scatter(x_coords_metres,
y_coords_metres,
s=marker_size,
c=field_values,
marker=MARKER_TYPE,
edgecolors=MARKER_EDGE_COLOUR,
linewidths=marker_edge_width,
cmap=colour_map,
vmin=colour_minimum,
vmax=colour_maximum)
def read_data_from_sparse_grid_file(netcdf_file_name, field_name_orig=None,
data_source=None, sentinel_values=None,
raise_error_if_fails=True):
"""Reads sparse radar grid from raw (either MYRORSS or MRMS) file.
This file should contain one radar field at one height and one time step.
:param netcdf_file_name: Path to input file.
:param field_name_orig: Name of radar field in original (either MYRORSS or
MRMS) format.
:param data_source: Data source (either "myrorss" or "mrms").
:param sentinel_values: 1-D numpy array of sentinel values.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error. If False and file cannot be
opened, this method will return None.
:return: sparse_grid_table: pandas DataFrame with the following columns.
Each row corresponds to one grid cell.
sparse_grid_table.grid_row: Row index.
sparse_grid_table.grid_column: Column index.
sparse_grid_table.<field_name>: Radar measurement (field_name is determined
by the method `field_name_orig_to_new`).
sparse_grid_table.num_grid_cells: Number of consecutive grid cells --
starting at the current one and counting along rows first, columns
second -- with the same radar measurement.
"""
error_checking.assert_file_exists(netcdf_file_name)
error_checking.assert_is_numpy_array_without_nan(sentinel_values)
error_checking.assert_is_numpy_array(sentinel_values, num_dimensions=1)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
field_name = _field_name_orig_to_new(field_name_orig,
data_source=data_source)
num_values = len(netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG])
if num_values == 0:
sparse_grid_dict = {
GRID_ROW_COLUMN: numpy.array([], dtype=int),
GRID_COLUMN_COLUMN: numpy.array([], dtype=int),
NUM_GRID_CELL_COLUMN: numpy.array([], dtype=int),
field_name: numpy.array([], dtype=int)}
else:
sparse_grid_dict = {
GRID_ROW_COLUMN: netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG][:],
GRID_COLUMN_COLUMN:
netcdf_dataset.variables[GRID_COLUMN_COLUMN_ORIG][:],
NUM_GRID_CELL_COLUMN:
netcdf_dataset.variables[NUM_GRID_CELL_COLUMN_ORIG][:],
field_name: netcdf_dataset.variables[field_name_orig][:]}
netcdf_dataset.close()
sparse_grid_table = pandas.DataFrame.from_dict(sparse_grid_dict)
return _remove_sentinels_from_sparse_grid(
sparse_grid_table, field_name, sentinel_values)
def plot_many_2d_grids(class_activation_matrix_3d,
axes_object_matrix,
colour_map_object,
min_contour_level,
max_contour_level,
contour_interval,
line_width=DEFAULT_CONTOUR_WIDTH,
row_major=True,
line_style=DEFAULT_CONTOUR_STYLE):
"""Plots the same 2-D class-activation map for each predictor.
M = number of rows in spatial grid
N = number of columns in spatial grid
P = number of predictors
:param class_activation_matrix_3d: M-by-N-by-P numpy array of class
activations.
:param axes_object_matrix: See doc for `plotting_utils.init_panels`.
:param colour_map_object: See doc for `plot_2d_grid`.
:param min_contour_level: Same.
:param max_contour_level: Same.
:param contour_interval: Same.
:param line_width: Same.
:param row_major: Boolean flag. If True, panels will be filled along rows
first, then down columns. If False, down columns first, then along
rows.
:param line_style: Style (e.g., "solid") for contour lines.
"""
error_checking.assert_is_numpy_array_without_nan(
class_activation_matrix_3d)
error_checking.assert_is_numpy_array(class_activation_matrix_3d,
num_dimensions=3)
error_checking.assert_is_boolean(row_major)
if row_major:
order_string = 'C'
else:
order_string = 'F'
num_predictors = class_activation_matrix_3d.shape[-1]
num_panel_rows = axes_object_matrix.shape[0]
num_panel_columns = axes_object_matrix.shape[1]
for k in range(num_predictors):
this_panel_row, this_panel_column = numpy.unravel_index(
k, (num_panel_rows, num_panel_columns), order=order_string)
plot_2d_grid(class_activation_matrix_2d=class_activation_matrix_3d[...,
k],
axes_object=axes_object_matrix[this_panel_row,
this_panel_column],
colour_map_object=colour_map_object,
min_contour_level=min_contour_level,
max_contour_level=max_contour_level,
contour_interval=contour_interval,
line_width=line_width,
line_style=line_style)
def get_thermal_front_param(thermal_field_matrix_kelvins, x_spacing_metres,
y_spacing_metres):
"""Computes thermal front parameter (TFP) at each grid point.
TFP is defined in Renard and Clarke (1965).
M = number of rows in grid
N = number of columns in grid
:param thermal_field_matrix_kelvins: M-by-N numpy array with values of
thermal variable. This can be any thermal variable ([potential]
temperature, wet-bulb [potential] temperature, equivalent [potential]
temperature, etc.).
:param x_spacing_metres: Spacing between grid points in adjacent columns.
:param y_spacing_metres: Spacing between grid points in adjacent rows.
:return: tfp_matrix_kelvins_m02: M-by-N numpy array with TFP at each grid
point. Units are Kelvins per m^2.
"""
error_checking.assert_is_numpy_array_without_nan(
thermal_field_matrix_kelvins)
error_checking.assert_is_greater_numpy_array(thermal_field_matrix_kelvins,
0.)
error_checking.assert_is_numpy_array(thermal_field_matrix_kelvins,
num_dimensions=2)
error_checking.assert_is_greater(x_spacing_metres, 0.)
error_checking.assert_is_greater(y_spacing_metres, 0.)
x_grad_matrix_kelvins_m01, y_grad_matrix_kelvins_m01 = _get_2d_gradient(
field_matrix=thermal_field_matrix_kelvins,
x_spacing_metres=x_spacing_metres,
y_spacing_metres=y_spacing_metres)
grad_magnitude_matrix_kelvins_m01 = numpy.sqrt(
x_grad_matrix_kelvins_m01**2 + y_grad_matrix_kelvins_m01**2)
(x_grad_grad_matrix_kelvins_m02,
y_grad_grad_matrix_kelvins_m02) = _get_2d_gradient(
field_matrix=grad_magnitude_matrix_kelvins_m01,
x_spacing_metres=x_spacing_metres,
y_spacing_metres=y_spacing_metres)
first_matrix = (-x_grad_grad_matrix_kelvins_m02 *
x_grad_matrix_kelvins_m01 /
grad_magnitude_matrix_kelvins_m01)
first_matrix[numpy.isnan(first_matrix)] = 0.
second_matrix = (-y_grad_grad_matrix_kelvins_m02 *
y_grad_matrix_kelvins_m01 /
grad_magnitude_matrix_kelvins_m01)
second_matrix[numpy.isnan(second_matrix)] = 0.
return first_matrix + second_matrix
def plot_2d_feature_map(
feature_matrix, axes_object, colour_map_object,
font_size=DEFAULT_FONT_SIZE, colour_norm_object=None,
min_colour_value=None, max_colour_value=None, annotation_string=None):
"""Plots 2-D feature map.
M = number of rows in grid
N = number of columns in grid
:param feature_matrix: M-by-N numpy array of feature values (either before
or after activation function -- this method doesn't care).
:param axes_object: Instance of `matplotlib.axes._subplots.AxesSubplot`.
:param font_size: Font size for annotation.
:param colour_map_object: Instance of `matplotlib.pyplot.cm`.
:param colour_norm_object: Instance of `matplotlib.colors.BoundaryNorm`.
:param min_colour_value: [used only if `colour_norm_object is None`]
Minimum value in colour scheme.
:param max_colour_value: [used only if `colour_norm_object is None`]
Max value in colour scheme.
:param annotation_string: Annotation (printed in the bottom-center of the
map). For no annotation, leave this alone.
"""
error_checking.assert_is_numpy_array_without_nan(feature_matrix)
error_checking.assert_is_numpy_array(feature_matrix, num_dimensions=2)
if colour_norm_object is None:
error_checking.assert_is_greater(max_colour_value, min_colour_value)
colour_norm_object = None
else:
if hasattr(colour_norm_object, 'boundaries'):
min_colour_value = colour_norm_object.boundaries[0]
max_colour_value = colour_norm_object.boundaries[-1]
else:
min_colour_value = colour_norm_object.vmin
max_colour_value = colour_norm_object.vmax
axes_object.pcolormesh(
feature_matrix, cmap=colour_map_object, norm=colour_norm_object,
vmin=min_colour_value, vmax=max_colour_value, shading='flat',
edgecolors='None')
if annotation_string is not None:
error_checking.assert_is_string(annotation_string)
axes_object.text(
0.5, 0.01, annotation_string, fontsize=font_size, fontweight='bold',
color='black', horizontalalignment='center',
verticalalignment='bottom', transform=axes_object.transAxes)
axes_object.set_xticks([])
axes_object.set_yticks([])
def _check_in_and_out_matrices(predictor_matrices,
num_examples=None,
saliency_matrices=None):
"""Error-checks input and output matrices.
T = number of input tensors to the model
E = number of examples (storm objects)
:param predictor_matrices: length-T list of predictor matrices. Each item
must be a numpy array.
:param num_examples: E in the above discussion. The first axis of each
array must have length E. If you don't know the number of examples,
leave this as None.
:param saliency_matrices: Same as `predictor_matrices` but with saliency
values.
:raises: ValueError: if `predictor_matrices` and `saliency_matrices` have
different lengths.
"""
error_checking.assert_is_list(predictor_matrices)
num_matrices = len(predictor_matrices)
if saliency_matrices is None:
saliency_matrices = [None] * num_matrices
error_checking.assert_is_list(saliency_matrices)
num_saliency_matrices = len(saliency_matrices)
if num_matrices != num_saliency_matrices:
error_string = (
'Number of predictor matrices ({0:d}) should = number of saliency '
'matrices ({1:d}).').format(num_matrices, num_saliency_matrices)
raise ValueError(error_string)
for i in range(num_matrices):
error_checking.assert_is_numpy_array_without_nan(predictor_matrices[i])
if num_examples is not None:
these_expected_dim = numpy.array(
(num_examples, ) + predictor_matrices[i].shape[1:], dtype=int)
error_checking.assert_is_numpy_array(
predictor_matrices[i], exact_dimensions=these_expected_dim)
if saliency_matrices[i] is not None:
error_checking.assert_is_numpy_array_without_nan(
saliency_matrices[i])
these_expected_dim = numpy.array(predictor_matrices[i].shape,
dtype=int)
error_checking.assert_is_numpy_array(
saliency_matrices[i], exact_dimensions=these_expected_dim)
def check_time_separation(unix_times_sec,
early_indices=None,
late_indices=None,
time_separation_sec=DEFAULT_TIME_SEPARATION_SEC):
"""Ensures that there is a separation (buffer) between two sets of times.
:param unix_times_sec: See documentation for _apply_time_separation.
:param early_indices: See documentation for _apply_time_separation.
:param late_indices: See documentation for _apply_time_separation.
:param time_separation_sec: See documentation for _apply_time_separation.
:raises: ValueError: if separation between sets is < `time_separation_sec`.
"""
error_checking.assert_is_integer_numpy_array(unix_times_sec)
error_checking.assert_is_numpy_array_without_nan(unix_times_sec)
error_checking.assert_is_numpy_array(unix_times_sec, num_dimensions=1)
num_times = len(unix_times_sec)
error_checking.assert_is_integer_numpy_array(early_indices)
error_checking.assert_is_numpy_array(early_indices, num_dimensions=1)
error_checking.assert_is_geq_numpy_array(early_indices, 0)
error_checking.assert_is_leq_numpy_array(early_indices, num_times - 1)
error_checking.assert_is_integer_numpy_array(late_indices)
error_checking.assert_is_numpy_array(late_indices, num_dimensions=1)
error_checking.assert_is_geq_numpy_array(late_indices, 0)
error_checking.assert_is_leq_numpy_array(late_indices, num_times - 1)
error_checking.assert_is_greater_numpy_array(
unix_times_sec[late_indices], numpy.max(unix_times_sec[early_indices]))
error_checking.assert_is_integer(time_separation_sec)
error_checking.assert_is_greater(time_separation_sec, 0)
last_early_time_unix_sec = numpy.max(unix_times_sec[early_indices])
first_late_time_unix_sec = numpy.min(unix_times_sec[late_indices])
min_diff_between_sets_sec = (first_late_time_unix_sec -
last_early_time_unix_sec)
if min_diff_between_sets_sec < time_separation_sec:
last_early_time_string = time_conversion.unix_sec_to_string(
last_early_time_unix_sec, TIME_STRING_FORMAT)
first_late_time_string = time_conversion.unix_sec_to_string(
first_late_time_unix_sec, TIME_STRING_FORMAT)
error_string = ('Last time in early set is ' + last_early_time_string +
'. First time in late set is ' +
first_late_time_string +
'. This is a time separation of ' +
str(min_diff_between_sets_sec) +
' seconds between sets. Required separation is >= ' +
str(time_separation_sec) + ' s.')
raise ValueError(error_string)
def geodetic_to_standard_angles(geodetic_angles_deg):
"""Converts angles from geodetic to standard format.
For the definitions of "geodetic format" and "standard format," see doc for
`standard_to_geodetic_angles`.
:param geodetic_angles_deg: numpy array of geodetic angles (degrees).
:return: standard_angles_deg: equivalent-size numpy array of standard
angles.
"""
error_checking.assert_is_numpy_array_without_nan(geodetic_angles_deg)
return numpy.mod((450. - geodetic_angles_deg), 360.)
def standard_to_geodetic_angles(standard_angles_deg):
"""Converts angles from standard to geodetic format.
"Standard format" = measured counterclockwise from due east
"Geodetic format" = measured clockwise from due north
:param standard_angles_deg: numpy array of standard angles (degrees).
:return: geodetic_angles_deg: equivalent-size numpy array of geodetic
angles.
"""
error_checking.assert_is_numpy_array_without_nan(standard_angles_deg)
return numpy.mod((450. - standard_angles_deg), 360.)
def get_front_types(locating_var_matrix_m01_s01,
warm_front_percentile=DEFAULT_FRONT_PERCENTILE,
cold_front_percentile=DEFAULT_FRONT_PERCENTILE):
"""Infers front type at each grid cell.
M = number of rows in grid
N = number of columns in grid
:param locating_var_matrix_m01_s01: M-by-N numpy array created by
`get_locating_variable`.
:param warm_front_percentile: Used to locate warm fronts. For grid cell
[i, j] to be considered part of a warm front, its locating value must be
<= the [q]th percentile of all non-positive values in the grid, where
q = `100 - warm_front_percentile`.
:param cold_front_percentile: Used to locate cold fronts. For grid cell
[i, j] to be considered part of a cold front, its locating value must be
>= the [q]th percentile of all non-negative values in the grid, where
q = `cold_front_percentile`.
:return: predicted_label_matrix: M-by-N numpy array, where the value at each
grid cell is from the list `front_utils.VALID_INTEGER_IDS`.
"""
error_checking.assert_is_numpy_array_without_nan(
locating_var_matrix_m01_s01)
error_checking.assert_is_numpy_array(locating_var_matrix_m01_s01,
num_dimensions=2)
error_checking.assert_is_greater(warm_front_percentile, 0.)
error_checking.assert_is_less_than(warm_front_percentile, 100.)
error_checking.assert_is_greater(cold_front_percentile, 0.)
error_checking.assert_is_less_than(cold_front_percentile, 100.)
warm_front_threshold_m01_s01 = numpy.percentile(
locating_var_matrix_m01_s01[locating_var_matrix_m01_s01 <= 0],
100 - warm_front_percentile)
cold_front_threshold_m01_s01 = numpy.percentile(
locating_var_matrix_m01_s01[locating_var_matrix_m01_s01 >= 0],
cold_front_percentile)
predicted_label_matrix = numpy.full(locating_var_matrix_m01_s01.shape,
front_utils.NO_FRONT_INTEGER_ID,
dtype=int)
predicted_label_matrix[
locating_var_matrix_m01_s01 <=
warm_front_threshold_m01_s01] = front_utils.WARM_FRONT_INTEGER_ID
predicted_label_matrix[
locating_var_matrix_m01_s01 >=
cold_front_threshold_m01_s01] = front_utils.COLD_FRONT_INTEGER_ID
return predicted_label_matrix
def do_2d_convolution(feature_matrix,
kernel_matrix,
pad_edges=False,
stride_length_px=1):
"""Convolves 2-D feature maps with 2-D kernel.
m = number of rows in kernel
n = number of columns in kernel
c = number of output feature maps (channels)
:param feature_matrix: Input feature maps (numpy array). Dimensions must be
M x N x C or 1 x M x N x C.
:param kernel_matrix: Kernel as numpy array. Dimensions must be
m x n x C x c.
:param pad_edges: Boolean flag. If True, edges of input feature maps will
be zero-padded during convolution, so spatial dimensions of the output
feature maps will be the same (M x N). If False, dimensions
of the output maps will be (M - m + 1) x (N - n + 1).
:param stride_length_px: Stride length (pixels). The kernel will move by
this many rows or columns at a time as it slides over each input feature
map.
:return: feature_matrix: Output feature maps (numpy array). Dimensions will
be 1 x M x N x c or 1 x (M - m + 1) x (N - n + 1) x c, depending on
whether or not edges are padded.
"""
error_checking.assert_is_numpy_array_without_nan(feature_matrix)
error_checking.assert_is_numpy_array_without_nan(kernel_matrix)
error_checking.assert_is_numpy_array(kernel_matrix, num_dimensions=4)
error_checking.assert_is_boolean(pad_edges)
error_checking.assert_is_integer(stride_length_px)
error_checking.assert_is_geq(stride_length_px, 1)
if len(feature_matrix.shape) == 3:
feature_matrix = numpy.expand_dims(feature_matrix, axis=0)
error_checking.assert_is_numpy_array(feature_matrix, num_dimensions=4)
if pad_edges:
padding_string = 'same'
else:
padding_string = 'valid'
feature_tensor = K.conv2d(x=K.variable(feature_matrix),
kernel=K.variable(kernel_matrix),
strides=(stride_length_px, stride_length_px),
padding=padding_string,
data_format='channels_last')
return feature_tensor.numpy()
def plot_many_2d_grids_with_contours(saliency_matrix_3d,
axes_object_matrix,
colour_map_object,
max_absolute_contour_level,
contour_interval,
line_width=DEFAULT_CONTOUR_WIDTH,
row_major=True):
"""Plots 2-D saliency map with line contours for each predictor.
M = number of rows in spatial grid
N = number of columns in spatial grid
P = number of predictors
:param saliency_matrix_3d: M-by-N-by-P numpy array of saliency values.
:param axes_object_matrix: See doc for
`plotting_utils.create_paneled_figure`.
:param colour_map_object: See doc for `plot_2d_grid_with_contours`.
:param max_absolute_contour_level: Same.
:param contour_interval: Same.
:param line_width: Same.
:param row_major: Boolean flag. If True, panels will be filled along rows
first, then down columns. If False, down columns first, then along
rows.
"""
error_checking.assert_is_numpy_array_without_nan(saliency_matrix_3d)
error_checking.assert_is_numpy_array(saliency_matrix_3d, num_dimensions=3)
error_checking.assert_is_boolean(row_major)
if row_major:
order_string = 'C'
else:
order_string = 'F'
num_predictors = saliency_matrix_3d.shape[-1]
num_panel_rows = axes_object_matrix.shape[0]
num_panel_columns = axes_object_matrix.shape[1]
for k in range(num_predictors):
this_panel_row, this_panel_column = numpy.unravel_index(
k, (num_panel_rows, num_panel_columns), order=order_string)
plot_2d_grid_with_contours(
saliency_matrix_2d=saliency_matrix_3d[..., k],
axes_object=axes_object_matrix[this_panel_row, this_panel_column],
colour_map_object=colour_map_object,
max_absolute_contour_level=max_absolute_contour_level,
contour_interval=contour_interval,
line_width=line_width)
def do_3d_convolution(feature_matrix,
kernel_matrix,
pad_edges=False,
stride_length_px=1):
"""Convolves 3-D feature maps with 3-D kernel.
m = number of rows in kernel
n = number of columns in kernel
h = number of height in kernel
c = number of output feature maps (channels)
:param feature_matrix: Input feature maps (numpy array). Dimensions must be
M x N x H x C or 1 x M x N x H x C.
:param kernel_matrix: Kernel as numpy array. Dimensions must be
m x n x h x C x c.
:param pad_edges: See doc for `do_2d_convolution`.
:param stride_length_px: See doc for `do_2d_convolution`.
:return: feature_matrix: Output feature maps (numpy array). Dimensions will
be 1 x M x N x H x c or
1 x (M - m + 1) x (N - n + 1) x (H - h + 1) x c, depending on
whether or not edges are padded.
"""
error_checking.assert_is_numpy_array_without_nan(feature_matrix)
error_checking.assert_is_numpy_array_without_nan(kernel_matrix)
error_checking.assert_is_numpy_array(kernel_matrix, num_dimensions=5)
error_checking.assert_is_boolean(pad_edges)
error_checking.assert_is_integer(stride_length_px)
error_checking.assert_is_geq(stride_length_px, 1)
if len(feature_matrix.shape) == 4:
feature_matrix = numpy.expand_dims(feature_matrix, axis=0)
error_checking.assert_is_numpy_array(feature_matrix, num_dimensions=5)
if pad_edges:
padding_string = 'same'
else:
padding_string = 'valid'
feature_tensor = K.conv3d(x=K.variable(feature_matrix),
kernel=K.variable(kernel_matrix),
strides=(stride_length_px, stride_length_px,
stride_length_px),
padding=padding_string,
data_format='channels_last')
return feature_tensor.numpy()
def geodetic_to_standard_angles(geodetic_angles_deg):
"""Converts angles from geodetic to standard format.
"Standard format" = measured counterclockwise from due east.
"Geodetic format" = measured clockwise from due north.
N = number of angles
:param geodetic_angles_deg: length-N numpy array of geodetic angles
(degrees).
:return: standard_angles_deg: length-N numpy array of standard angles
(degrees).
"""
error_checking.assert_is_numpy_array_without_nan(geodetic_angles_deg)
return numpy.mod((450. - geodetic_angles_deg), 360.)
def _check_polyline(x_coords_metres, y_coords_metres):
"""Checks polyline for errors.
V = number of vertices
:param x_coords_metres: length-V numpy array of x-coordinates.
:param y_coords_metres: length-V numpy array of y-coordinates.
"""
error_checking.assert_is_numpy_array_without_nan(x_coords_metres)
error_checking.assert_is_numpy_array(x_coords_metres, num_dimensions=1)
num_vertices = len(x_coords_metres)
error_checking.assert_is_numpy_array_without_nan(y_coords_metres)
error_checking.assert_is_numpy_array(y_coords_metres,
exact_dimensions=numpy.array(
[num_vertices]))
