def check_metadata(layer_name, neuron_indices, ideal_activation,
num_iterations, learning_rate, l2_weight):
"""Checks metadata for errors.
:param layer_name: Name of layer with relevant neuron.
:param neuron_indices: 1-D numpy array with indices of relevant neuron.
Must have length D - 1, where D = number of dimensions in layer output.
The first dimension is the batch dimension, which always has length
`None` in Keras.
:param ideal_activation: Ideal neuron activation, used to define loss
function. The loss function will be
(neuron_activation - ideal_activation)**2.
:param num_iterations: Number of iterations for gradient descent.
:param learning_rate: Learning rate for gradient descent.
:param l2_weight: L2 weight (penalty for difference between initial and
final predictor matrix) in loss function.
"""
error_checking.assert_is_string(layer_name)
error_checking.assert_is_integer_numpy_array(neuron_indices)
error_checking.assert_is_geq_numpy_array(neuron_indices, 0)
error_checking.assert_is_numpy_array(neuron_indices, num_dimensions=1)
error_checking.assert_is_not_nan(ideal_activation)
error_checking.assert_is_integer(num_iterations)
error_checking.assert_is_greater(num_iterations, 0)
error_checking.assert_is_greater(learning_rate, 0.)
error_checking.assert_is_less_than(learning_rate, 1.)
error_checking.assert_is_geq(l2_weight, 0.)
def check_metadata(activation_layer_name, vector_output_layer_name,
output_neuron_indices, ideal_activation):
"""Checks metadata for errors.
:param activation_layer_name: Name of activation layer.
:param vector_output_layer_name: Name of layer that outputs predictions for
vector target variables.
:param output_neuron_indices: length-2 numpy array with indices of output
neuron (height index, channel index). Class activation will be computed
with respect to the output of this neuron.
:param ideal_activation: Ideal neuron activation, used to define loss
function. The loss function will be
(output_neuron_activation - ideal_activation)**2.
"""
error_checking.assert_is_string(activation_layer_name)
error_checking.assert_is_string(vector_output_layer_name)
error_checking.assert_is_integer_numpy_array(output_neuron_indices)
error_checking.assert_is_geq_numpy_array(output_neuron_indices, 0)
error_checking.assert_is_numpy_array(output_neuron_indices,
exact_dimensions=numpy.array(
[2], dtype=int))
error_checking.assert_is_not_nan(ideal_activation)
def read_5minute_winds_from_raw_file(text_file_name, utc_offset_hours):
"""Reads 5-minute wind observations from raw file.
This file should contain 5-minute METARs for one station-month (see
download_5minute_file).
:param text_file_name: Path to input file.
:param utc_offset_hours: Difference between local station time and UTC
(local minus UTC).
:return: wind_table: pandas DataFrame with the following columns.
wind_table.unix_time_sec: Observation time (seconds since 0000 UTC 1 Jan
1970).
wind_table.wind_speed_m_s01: Speed of sustained wind (m/s).
wind_table.wind_direction_deg: Direction of sustained wind (degrees of
origin -- i.e., direction that the wind is coming from -- as per
meteorological convention).
wind_table.wind_gust_speed_m_s01: Speed of wind gust (m/s).
wind_table.wind_gust_direction_deg: Direction of wind gust (degrees of
origin).
"""
error_checking.assert_file_exists(text_file_name)
error_checking.assert_is_not_nan(utc_offset_hours)
unix_times_sec = []
wind_speeds_m_s01 = []
wind_directions_deg = []
wind_gust_speeds_m_s01 = []
wind_gust_directions_deg = []
for this_line in open(text_file_name, 'r').readlines():
this_local_time_string = (
this_line[LOCAL_TIME_CHAR_INDICES_5MINUTE_FILE[0]:
LOCAL_TIME_CHAR_INDICES_5MINUTE_FILE[1]])
this_time_unix_sec = _local_time_string_to_unix_sec(
this_local_time_string, utc_offset_hours)
(this_wind_speed_m_s01, this_wind_direction_deg,
this_wind_gust_speed_m_s01, this_wind_gust_direction_deg
) = _parse_5minute_wind_from_line(this_line)
unix_times_sec.append(this_time_unix_sec)
wind_speeds_m_s01.append(this_wind_speed_m_s01)
wind_directions_deg.append(this_wind_direction_deg)
wind_gust_speeds_m_s01.append(this_wind_gust_speed_m_s01)
wind_gust_directions_deg.append(this_wind_gust_direction_deg)
wind_dict = {
raw_wind_io.WIND_SPEED_COLUMN: wind_speeds_m_s01,
raw_wind_io.WIND_DIR_COLUMN: wind_directions_deg,
raw_wind_io.WIND_GUST_SPEED_COLUMN: wind_gust_speeds_m_s01,
raw_wind_io.WIND_GUST_DIR_COLUMN: wind_gust_directions_deg,
raw_wind_io.TIME_COLUMN: unix_times_sec
}
wind_table = pandas.DataFrame.from_dict(wind_dict)
wind_table[raw_wind_io.WIND_SPEED_COLUMN] *= KT_TO_METRES_PER_SECOND
wind_table[raw_wind_io.WIND_GUST_SPEED_COLUMN] *= KT_TO_METRES_PER_SECOND
return _remove_invalid_wind_rows(wind_table)
def project_latlng_to_xy(latitudes_deg,
longitudes_deg,
projection_object=None,
false_easting_metres=0.,
false_northing_metres=0.):
"""Converts from lat-long to projection (x-y) coordinates.
S = shape of coordinate arrays.
:param latitudes_deg: numpy array of latitudes (deg N) with shape S.
:param longitudes_deg: numpy array of longitudes (deg E) with shape S.
:param projection_object: Projection object created by `pyproj.Proj`.
:param false_easting_metres: False easting. Will be added to all x-
coordinates.
:param false_northing_metres: False northing. Will be added to all y-
coordinates.
:return: x_coords_metres: numpy array of x-coordinates with shape S.
:return: y_coords_metres: numpy array of y-coordinates with shape S.
"""
error_checking.assert_is_valid_lat_numpy_array(latitudes_deg,
allow_nan=True)
shape_of_coord_arrays = latitudes_deg.shape
error_checking.assert_is_numpy_array(
longitudes_deg, exact_dimensions=numpy.asarray(shape_of_coord_arrays))
longitudes_deg = lng_conversion.convert_lng_positive_in_west(
longitudes_deg, allow_nan=True)
error_checking.assert_is_not_nan(false_easting_metres)
error_checking.assert_is_not_nan(false_northing_metres)
x_coords_metres, y_coords_metres = projection_object(
longitudes_deg, latitudes_deg)
num_points = latitudes_deg.size
nan_flags = numpy.logical_or(
numpy.isnan(numpy.reshape(latitudes_deg, num_points)),
numpy.isnan(numpy.reshape(longitudes_deg, num_points)))
nan_indices = numpy.where(nan_flags)[0]
x_coords_metres_flat = numpy.reshape(x_coords_metres, num_points)
x_coords_metres_flat[nan_indices] = numpy.nan
x_coords_metres = numpy.reshape(
x_coords_metres_flat, shape_of_coord_arrays) + false_easting_metres
y_coords_metres_flat = numpy.reshape(y_coords_metres, num_points)
y_coords_metres_flat[nan_indices] = numpy.nan
y_coords_metres = numpy.reshape(
y_coords_metres_flat, shape_of_coord_arrays) + false_northing_metres
return x_coords_metres, y_coords_metres
def project_xy_to_latlng(x_coords_metres,
y_coords_metres,
projection_object=None,
false_easting_metres=0.,
false_northing_metres=0.):
"""Converts from projection (x-y) to lat-long coordinates.
P = number of points
:param x_coords_metres: length-P numpy array of x-coordinates.
:param y_coords_metres: length-P numpy array of y-coordinates.
:param projection_object: Projection object created by `pyproj.Proj`.
:param false_easting_metres: False easting. Will be subtracted from all x-
coordinates before conversion.
:param false_northing_metres: False northing. Will be subtracted from all
y-coordinates before conversion.
:return: latitudes_deg: length-P numpy array of latitudes (deg N).
:return: longitudes_deg: length-P numpy array of longitudes (deg E).
"""
error_checking.assert_is_real_numpy_array(x_coords_metres)
shape_of_coord_arrays = x_coords_metres.shape
error_checking.assert_is_numpy_array(
y_coords_metres, exact_dimensions=numpy.asarray(shape_of_coord_arrays))
error_checking.assert_is_real_numpy_array(y_coords_metres)
error_checking.assert_is_not_nan(false_easting_metres)
error_checking.assert_is_not_nan(false_northing_metres)
(longitudes_deg, latitudes_deg) = projection_object(
x_coords_metres - false_easting_metres,
y_coords_metres - false_northing_metres,
inverse=True)
num_points = x_coords_metres.size
nan_flags = numpy.logical_or(
numpy.isnan(numpy.reshape(x_coords_metres, num_points)),
numpy.isnan(numpy.reshape(y_coords_metres, num_points)))
nan_indices = numpy.where(nan_flags)[0]
latitudes_deg_flat = numpy.reshape(latitudes_deg, num_points)
latitudes_deg_flat[nan_indices] = numpy.nan
longitudes_deg_flat = numpy.reshape(longitudes_deg, num_points)
longitudes_deg_flat[nan_indices] = numpy.nan
longitudes_deg_flat = lng_conversion.convert_lng_positive_in_west(
longitudes_deg_flat, allow_nan=True)
return (numpy.reshape(latitudes_deg_flat, shape_of_coord_arrays),
numpy.reshape(longitudes_deg_flat, shape_of_coord_arrays))
def time_and_period_length_to_range(unix_time_sec, period_length_sec):
"""Converts single time and period length to range (start/end of period).
:param unix_time_sec: Single time (Unix format).
:param period_length_sec: Length of time period (seconds).
:return: start_time_unix_sec: Beginning of time period (Unix format).
:return: end_time_unix_sec: End of time period (Unix format).
"""
error_checking.assert_is_integer(unix_time_sec)
error_checking.assert_is_not_nan(unix_time_sec)
error_checking.assert_is_integer(period_length_sec)
start_time_unix_sec = int(rounder.floor_to_nearest(
float(unix_time_sec), period_length_sec))
return start_time_unix_sec, start_time_unix_sec + period_length_sec
def is_time_in_spc_date(unix_time_sec, spc_date_string):
"""Determines whether or not time is in SPC date.
:param unix_time_sec: Time in Unix format.
:param spc_date_string: SPC date in format "yyyymmdd".
:return: time_in_spc_date_flag: Boolean flag.
"""
min_time_unix_sec = get_start_of_spc_date(spc_date_string)
max_time_unix_sec = get_end_of_spc_date(spc_date_string)
error_checking.assert_is_integer(unix_time_sec)
error_checking.assert_is_not_nan(unix_time_sec)
return numpy.logical_and(unix_time_sec >= min_time_unix_sec,
unix_time_sec <= max_time_unix_sec)
def get_xy_grid_points(x_min_metres=None,
y_min_metres=None,
x_spacing_metres=None,
y_spacing_metres=None,
num_rows=None,
num_columns=None):
"""Generates unique x- and y-coords of grid points in regular x-y grid.
M = number of rows in grid
N = number of columns in grid
:param x_min_metres: Minimum x-coordinate over all grid points.
:param y_min_metres: Minimum y-coordinate over all grid points.
:param x_spacing_metres: Spacing between adjacent grid points in x-
direction. Alternate interpretation: length of each grid cell in x-
direction.
:param y_spacing_metres: Spacing between adjacent grid points in y-
direction. Alternate interpretation: length of each grid cell in y-
direction.
:param num_rows: Number of rows (unique grid-point y-values) in grid.
:param num_columns: Number of columns (unique grid-point x-values) in grid.
:return: grid_point_x_metres: length-N numpy array with x-coordinates of
grid points.
:return: grid_point_y_metres: length-M numpy array with y-coordinates of
grid points.
"""
error_checking.assert_is_not_nan(x_min_metres)
error_checking.assert_is_not_nan(y_min_metres)
error_checking.assert_is_greater(x_spacing_metres, 0.)
error_checking.assert_is_greater(y_spacing_metres, 0.)
error_checking.assert_is_integer(num_rows)
error_checking.assert_is_greater(num_rows, 0)
error_checking.assert_is_integer(num_columns)
error_checking.assert_is_greater(num_columns, 0)
x_max_metres = x_min_metres + (num_columns - 1) * x_spacing_metres
y_max_metres = y_min_metres + (num_rows - 1) * y_spacing_metres
grid_point_x_metres = numpy.linspace(x_min_metres,
x_max_metres,
num=num_columns)
grid_point_y_metres = numpy.linspace(y_min_metres,
y_max_metres,
num=num_rows)
return grid_point_x_metres, grid_point_y_metres
def range_and_interval_to_list(start_time_unix_sec=None,
end_time_unix_sec=None,
time_interval_sec=None,
include_endpoint=True):
"""Converts time period from range and interval to list of exact times.
N = number of exact times
:param start_time_unix_sec: Start time (Unix format).
:param end_time_unix_sec: End time (Unix format).
:param time_interval_sec: Interval (seconds) between successive exact times.
:param include_endpoint: Boolean flag. If True, endpoint will be included
in list of time steps. If False, endpoint will be excluded.
:return: unix_times_sec: length-N numpy array of exact times (Unix format).
"""
error_checking.assert_is_integer(start_time_unix_sec)
error_checking.assert_is_not_nan(start_time_unix_sec)
error_checking.assert_is_integer(end_time_unix_sec)
error_checking.assert_is_not_nan(end_time_unix_sec)
error_checking.assert_is_integer(time_interval_sec)
error_checking.assert_is_boolean(include_endpoint)
if include_endpoint:
error_checking.assert_is_geq(end_time_unix_sec, start_time_unix_sec)
else:
error_checking.assert_is_greater(end_time_unix_sec,
start_time_unix_sec)
start_time_unix_sec = int(
rounder.floor_to_nearest(float(start_time_unix_sec),
time_interval_sec))
end_time_unix_sec = int(
rounder.ceiling_to_nearest(float(end_time_unix_sec),
time_interval_sec))
if not include_endpoint:
end_time_unix_sec -= time_interval_sec
num_time_steps = 1 + int(
numpy.round(
(end_time_unix_sec - start_time_unix_sec) / time_interval_sec))
return numpy.linspace(start_time_unix_sec,
end_time_unix_sec,
num=num_time_steps,
dtype=int)
def radar_field_and_statistic_to_column_name(radar_field_name,
radar_height_m_asl,
statistic_name):
"""Generates column name for radar field and statistic.
:param radar_field_name: Name of radar field.
:param radar_height_m_asl: Radar height (metres above sea level).
:param statistic_name: Name of statistic.
:return: column_name: Name of column.
"""
error_checking.assert_is_string(radar_field_name)
error_checking.assert_is_not_nan(radar_height_m_asl)
error_checking.assert_is_string(statistic_name)
return '{0:s}_{1:d}metres_{2:s}'.format(
radar_field_name, int(numpy.round(radar_height_m_asl)), statistic_name)
def check_input_args(input_matrix, max_percentile_level, threshold_var_index,
threshold_value, threshold_type_string):
"""Error-checks input arguments.
:param input_matrix: See doc for `run_pmm_many_variables`.
:param max_percentile_level: Same.
:param threshold_var_index: Same.
:param threshold_value: Same.
:param threshold_type_string: Same.
:return: metadata_dict: Dictionary with the following keys.
metadata_dict['max_percentile_level']: See input doc.
metadata_dict['threshold_var_index']: See input doc.
metadata_dict['threshold_value']: See input doc.
metadata_dict['threshold_type_string']: See input doc.
"""
error_checking.assert_is_numpy_array_without_nan(input_matrix)
num_spatial_dimensions = len(input_matrix.shape) - 2
error_checking.assert_is_geq(num_spatial_dimensions, 1)
error_checking.assert_is_greater(max_percentile_level, 50.)
error_checking.assert_is_leq(max_percentile_level, 100.)
use_threshold = not (threshold_var_index is None
and threshold_value is None
and threshold_type_string is None)
if use_threshold:
_check_threshold_type(threshold_type_string)
error_checking.assert_is_not_nan(threshold_value)
error_checking.assert_is_integer(threshold_var_index)
error_checking.assert_is_geq(threshold_var_index, 0)
num_variables = input_matrix.shape[-1]
error_checking.assert_is_less_than(threshold_var_index, num_variables)
else:
threshold_var_index = -1
return {
MAX_PERCENTILE_KEY: max_percentile_level,
THRESHOLD_VAR_KEY: threshold_var_index,
THRESHOLD_VALUE_KEY: threshold_value,
THRESHOLD_TYPE_KEY: threshold_type_string
}
def radar_field_and_percentile_to_column_name(radar_field_name,
radar_height_m_asl,
percentile_level):
"""Generates column name for radar field and percentile level.
:param radar_field_name: Name of radar field.
:param radar_height_m_asl: Radar height (metres above sea level).
:param percentile_level: Percentile level.
:return: column_name: Name of column.
"""
error_checking.assert_is_string(radar_field_name)
error_checking.assert_is_not_nan(radar_height_m_asl)
error_checking.assert_is_not_nan(percentile_level)
return '{0:s}_{1:d}metres_percentile{2:05.1f}'.format(
radar_field_name, int(numpy.round(radar_height_m_asl)),
percentile_level)
def is_time_in_spc_date(unix_time_sec, spc_date_string):
"""Determines whether or not time is in SPC date.
:param unix_time_sec: Time in Unix format.
:param spc_date_string: SPC date in format "yyyymmdd".
:return: time_in_spc_date_flag: Boolean flag.
"""
min_time_unix_sec = MIN_SECONDS_INTO_SPC_DATE + string_to_unix_sec(
spc_date_string, SPC_DATE_FORMAT)
max_time_unix_sec = MAX_SECONDS_INTO_SPC_DATE + string_to_unix_sec(
spc_date_string, SPC_DATE_FORMAT)
error_checking.assert_is_integer(unix_time_sec)
error_checking.assert_is_not_nan(unix_time_sec)
return numpy.logical_and(unix_time_sec >= min_time_unix_sec,
unix_time_sec <= max_time_unix_sec)
def check_metadata(layer_name, neuron_indices, ideal_activation):
"""Checks metadata for errors.
The "relevant neuron" is that whose activation will be used in the numerator
of the saliency equation. In other words, if the relevant neuron is n,
the saliency of each predictor x will be d(a_n) / dx, where a_n is the
activation of n.
:param layer_name: Name of layer with relevant neuron.
:param neuron_indices: 1-D numpy array with indices of relevant neuron.
Must have length D - 1, where D = number of dimensions in layer output.
The first dimension is the batch dimension, which always has length
`None` in Keras.
:param ideal_activation: Ideal neuron activation, used to define loss
function. The loss function will be
(neuron_activation - ideal_activation)**2.
"""
error_checking.assert_is_string(layer_name)
error_checking.assert_is_integer_numpy_array(neuron_indices)
error_checking.assert_is_geq_numpy_array(neuron_indices, 0)
error_checking.assert_is_numpy_array(neuron_indices, num_dimensions=1)
error_checking.assert_is_not_nan(ideal_activation)
def point_in_or_on_polygon(polygon_object, query_x_coordinate,
query_y_coordinate):
"""Returns True if point is inside/touching the polygon, False otherwise.
x- and y-coordinates may be in one of three formats (see docstring at top of
file). However, the 3 input arguments must have coordinates in the same
format.
:param polygon_object: `shapely.geometry.Polygon` object.
:param query_x_coordinate: x-coordinate of query point.
:param query_y_coordinate: y-coordinate of query point.
:return: result: Boolean flag. True if point is inside/touching the
polygon, False otherwise.
"""
error_checking.assert_is_not_nan(query_x_coordinate)
error_checking.assert_is_not_nan(query_y_coordinate)
point_object = shapely.geometry.Point(query_x_coordinate,
query_y_coordinate)
if polygon_object.contains(point_object):
return True
return polygon_object.touches(point_object)
def test_assert_is_not_nan_none(self):
"""Checks assert_is_not_nan when input is None."""
with self.assertRaises(TypeError):
error_checking.assert_is_not_nan(None)
def read_polygons_from_netcdf(netcdf_file_name,
metadata_dict,
spc_date_string,
tracking_start_time_unix_sec,
tracking_end_time_unix_sec,
raise_error_if_fails=True):
"""Reads storm polygons (outlines of storm cells) from NetCDF file.
P = number of grid points in storm cell (different for each storm cell)
V = number of vertices in storm polygon (different for each storm cell)
If file cannot be opened, returns None.
:param netcdf_file_name: Path to input file.
:param metadata_dict: Dictionary with metadata for NetCDF file, created by
`myrorss_and_mrms_io.read_metadata_from_raw_file`.
:param spc_date_string: SPC date (format "yyyymmdd").
:param tracking_start_time_unix_sec: Start time for tracking period. This
can be found by `get_start_end_times_for_spc_date`.
:param tracking_end_time_unix_sec: End time for tracking period. This can
be found by `get_start_end_times_for_spc_date`.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error.
:return: polygon_table: pandas DataFrame with the following columns. Each
row is one storm object.
polygon_table.primary_id_string: See documentation for
`storm_tracking_io.write_file`.
polygon_table.valid_time_unix_sec: Same.
polygon_table.spc_date_string: Same.
polygon_table.tracking_start_time_unix_sec: Same.
polygon_table.tracking_end_time_unix_sec: Same.
polygon_table.centroid_latitude_deg: Same.
polygon_table.centroid_longitude_deg: Same.
polygon_table.grid_point_latitudes_deg: Same.
polygon_table.grid_point_longitudes_deg: Same.
polygon_table.grid_point_rows: Same.
polygon_table.grid_point_columns: Same.
polygon_table.polygon_object_latlng_deg: Same.
polygon_table.polygon_object_rowcol: Same.
"""
error_checking.assert_file_exists(netcdf_file_name)
error_checking.assert_is_integer(tracking_start_time_unix_sec)
error_checking.assert_is_not_nan(tracking_start_time_unix_sec)
error_checking.assert_is_integer(tracking_end_time_unix_sec)
error_checking.assert_is_not_nan(tracking_end_time_unix_sec)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
storm_id_column = metadata_dict[radar_utils.FIELD_NAME_COLUMN]
storm_id_column_orig = metadata_dict[
myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG]
num_values = len(
netcdf_dataset.variables[myrorss_and_mrms_io.GRID_ROW_COLUMN_ORIG])
if num_values == 0:
sparse_grid_dict = {
myrorss_and_mrms_io.GRID_ROW_COLUMN: numpy.array([], dtype=int),
myrorss_and_mrms_io.GRID_COLUMN_COLUMN: numpy.array([], dtype=int),
myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN: numpy.array([],
dtype=int),
storm_id_column: numpy.array([], dtype=int)
}
else:
sparse_grid_dict = {
myrorss_and_mrms_io.GRID_ROW_COLUMN:
netcdf_dataset.variables[myrorss_and_mrms_io.GRID_ROW_COLUMN_ORIG]
[:],
myrorss_and_mrms_io.GRID_COLUMN_COLUMN:
netcdf_dataset.variables[
myrorss_and_mrms_io.GRID_COLUMN_COLUMN_ORIG][:],
myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN:
netcdf_dataset.variables[
myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN_ORIG][:],
storm_id_column:
netcdf_dataset.variables[storm_id_column_orig][:]
}
netcdf_dataset.close()
sparse_grid_table = pandas.DataFrame.from_dict(sparse_grid_dict)
numeric_id_matrix = radar_s2f.sparse_to_full_grid(sparse_grid_table,
metadata_dict)[0]
polygon_table = _id_matrix_to_coord_lists(numeric_id_matrix)
num_storms = len(polygon_table.index)
valid_times_unix_sec = numpy.full(
num_storms, metadata_dict[radar_utils.UNIX_TIME_COLUMN], dtype=int)
spc_date_strings = num_storms * [
time_conversion.time_to_spc_date_string(valid_times_unix_sec[0])
]
tracking_start_times_unix_sec = numpy.full(num_storms,
tracking_start_time_unix_sec,
dtype=int)
tracking_end_times_unix_sec = numpy.full(num_storms,
tracking_end_time_unix_sec,
dtype=int)
simple_array = numpy.full(num_storms, numpy.nan)
object_array = numpy.full(num_storms, numpy.nan, dtype=object)
nested_array = polygon_table[[
tracking_utils.PRIMARY_ID_COLUMN, tracking_utils.PRIMARY_ID_COLUMN
]].values.tolist()
argument_dict = {
tracking_utils.VALID_TIME_COLUMN: valid_times_unix_sec,
tracking_utils.SPC_DATE_COLUMN: spc_date_strings,
tracking_utils.TRACKING_START_TIME_COLUMN:
tracking_start_times_unix_sec,
tracking_utils.TRACKING_END_TIME_COLUMN: tracking_end_times_unix_sec,
tracking_utils.CENTROID_LATITUDE_COLUMN: simple_array,
tracking_utils.CENTROID_LONGITUDE_COLUMN: simple_array,
tracking_utils.LATITUDES_IN_STORM_COLUMN: nested_array,
tracking_utils.LONGITUDES_IN_STORM_COLUMN: nested_array,
tracking_utils.LATLNG_POLYGON_COLUMN: object_array,
tracking_utils.ROWCOL_POLYGON_COLUMN: object_array
}
polygon_table = polygon_table.assign(**argument_dict)
for i in range(num_storms):
these_vertex_rows, these_vertex_columns = (
polygons.grid_points_in_poly_to_vertices(
grid_point_row_indices=polygon_table[
tracking_utils.ROWS_IN_STORM_COLUMN].values[i],
grid_point_column_indices=polygon_table[
tracking_utils.COLUMNS_IN_STORM_COLUMN].values[i]))
(polygon_table[tracking_utils.ROWS_IN_STORM_COLUMN].values[i],
polygon_table[tracking_utils.COLUMNS_IN_STORM_COLUMN].values[i]
) = polygons.simple_polygon_to_grid_points(
vertex_row_indices=these_vertex_rows,
vertex_column_indices=these_vertex_columns)
(polygon_table[tracking_utils.LATITUDES_IN_STORM_COLUMN].values[i],
polygon_table[tracking_utils.LONGITUDES_IN_STORM_COLUMN].values[i]
) = radar_utils.rowcol_to_latlng(
grid_rows=polygon_table[
tracking_utils.ROWS_IN_STORM_COLUMN].values[i],
grid_columns=polygon_table[
tracking_utils.COLUMNS_IN_STORM_COLUMN].values[i],
nw_grid_point_lat_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN])
these_vertex_lat_deg, these_vertex_lng_deg = (
radar_utils.rowcol_to_latlng(
grid_rows=these_vertex_rows,
grid_columns=these_vertex_columns,
nw_grid_point_lat_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN]))
(polygon_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values[i],
polygon_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values[i]
) = geodetic_utils.get_latlng_centroid(
latitudes_deg=these_vertex_lat_deg,
longitudes_deg=these_vertex_lng_deg)
polygon_table[tracking_utils.ROWCOL_POLYGON_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(
exterior_x_coords=these_vertex_columns,
exterior_y_coords=these_vertex_rows))
polygon_table[tracking_utils.LATLNG_POLYGON_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(
exterior_x_coords=these_vertex_lng_deg,
exterior_y_coords=these_vertex_lat_deg))
primary_id_strings = _append_spc_date_to_storm_ids(
primary_id_strings=polygon_table[
tracking_utils.PRIMARY_ID_COLUMN].values,
spc_date_string=spc_date_string)
return polygon_table.assign(
**{tracking_utils.PRIMARY_ID_COLUMN: primary_id_strings})
def plot_one_example(list_of_predictor_matrices,
model_metadata_dict,
plot_sounding=True,
allow_whitespace=True,
pmm_flag=False,
example_index=None,
full_storm_id_string=None,
storm_time_unix_sec=None,
storm_activation=None):
"""Plots predictors for one example.
R = number of radar figures
:param list_of_predictor_matrices: List created by
`testing_io.read_specific_examples`.
:param model_metadata_dict: Dictionary returned by
`cnn.read_model_metadata`.
:param plot_sounding: See documentation at top of file.
:param allow_whitespace: Same.
:param pmm_flag: Boolean flag. If True, will plot PMM (probability-matched
mean) composite of many examples (storm objects). If False, will plot
one example.
:param example_index: [used only if `pmm_flag == False`]
Will plot the [i]th example, where i = `example_index`.
:param full_storm_id_string: [used only if `pmm_flag == False`]
Full storm ID.
:param storm_time_unix_sec: [used only if `pmm_flag == False`]
Storm time.
:param storm_activation: [used only if `pmm_flag == False`]
Model activation for this example. Even if `pmm_flag == True`, this may
be None.
:return: handle_dict: Dictionary with the following keys.
handle_dict['sounding_figure_object']: One figure handle (instance of
`matplotlib.figure.Figure`). If sounding was not plotted, this is None.
handle_dict['sounding_axes_object']: One axes handle (instance of
`matplotlib.axes._subplots.AxesSubplot`). If sounding was not plotted,
this is None.
handle_dict['radar_figure_objects']: length-R list of figure handles
(instances of `matplotlib.figure.Figure`).
handle_dict['radar_axes_object_matrices']: length-R list. Each element is a
2-D numpy array of axes handles (instances of
`matplotlib.axes._subplots.AxesSubplot`).
"""
error_checking.assert_is_boolean(plot_sounding)
error_checking.assert_is_boolean(allow_whitespace)
error_checking.assert_is_boolean(pmm_flag)
if pmm_flag:
title_string = 'PMM composite'
if list_of_predictor_matrices[0].shape[0] == 1:
predictor_matrices_to_plot = [
a[0, ...] for a in list_of_predictor_matrices
]
else:
predictor_matrices_to_plot = list_of_predictor_matrices
else:
error_checking.assert_is_integer(example_index)
predictor_matrices_to_plot = [
a[example_index, ...] for a in list_of_predictor_matrices
]
error_checking.assert_is_string(full_storm_id_string)
storm_time_string = time_conversion.unix_sec_to_string(
storm_time_unix_sec, TIME_FORMAT)
if storm_activation is not None:
error_checking.assert_is_not_nan(storm_activation)
title_string = 'Storm "{0:s}" at {1:s}'.format(full_storm_id_string,
storm_time_string)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
has_sounding = (training_option_dict[trainval_io.SOUNDING_FIELDS_KEY]
is not None)
if plot_sounding and has_sounding:
sounding_figure_object, sounding_axes_object = _plot_sounding(
list_of_predictor_matrices=predictor_matrices_to_plot,
model_metadata_dict=model_metadata_dict,
allow_whitespace=allow_whitespace,
title_string=title_string)
else:
sounding_figure_object = None
sounding_axes_object = None
num_radar_matrices = len(predictor_matrices_to_plot) - int(has_sounding)
num_radar_dimensions = len(predictor_matrices_to_plot[0].shape) - 1
if num_radar_matrices == 2:
radar_figure_objects, radar_axes_object_matrices = (
_plot_2d3d_radar_scan(
list_of_predictor_matrices=predictor_matrices_to_plot,
model_metadata_dict=model_metadata_dict,
allow_whitespace=allow_whitespace,
title_string=title_string))
elif num_radar_dimensions == 3:
radar_figure_objects, radar_axes_object_matrices = _plot_3d_radar_scan(
list_of_predictor_matrices=predictor_matrices_to_plot,
model_metadata_dict=model_metadata_dict,
allow_whitespace=allow_whitespace,
title_string=title_string)
else:
radar_figure_objects, radar_axes_object_matrices = _plot_2d_radar_scan(
list_of_predictor_matrices=predictor_matrices_to_plot,
model_metadata_dict=model_metadata_dict,
allow_whitespace=allow_whitespace,
title_string=title_string)
return {
SOUNDING_FIGURE_KEY: sounding_figure_object,
SOUNDING_AXES_KEY: sounding_axes_object,
RADAR_FIGURES_KEY: radar_figure_objects,
RADAR_AXES_KEY: radar_axes_object_matrices
}
def read_field_from_grib_file(grib_file_name,
field_name_grib1,
num_grid_rows,
num_grid_columns,
sentinel_value=None,
temporary_dir_name=None,
wgrib_exe_name=WGRIB_EXE_NAME_DEFAULT,
wgrib2_exe_name=WGRIB2_EXE_NAME_DEFAULT,
raise_error_if_fails=True):
"""Reads field from grib file.
One field = one variable at one time step.
M = number of rows (unique y-coordinates or latitudes of grid points)
N = number of columns (unique x-coordinates or longitudes of grid points)
:param grib_file_name: Path to input file.
:param field_name_grib1: Field name in grib1 format (example: 500-mb height
is "HGT:500 mb").
:param num_grid_rows: Number of rows expected in grid.
:param num_grid_columns: Number of columns expected in grid.
:param sentinel_value: Sentinel value (all instances will be replaced with
NaN).
:param temporary_dir_name: Name of temporary directory. An intermediate
text file will be stored here.
:param wgrib_exe_name: Path to wgrib executable.
:param wgrib2_exe_name: Path to wgrib2 executable.
:param raise_error_if_fails: Boolean flag. If the extraction fails and
raise_error_if_fails = True, this method will error out. If the
extraction fails and raise_error_if_fails = False, this method will
return None.
:return: field_matrix: M-by-N numpy array with values of the given field.
If the grid is regular in x-y coordinates, x increases towards the right
(in the positive direction of the second axis), while y increases
downward (in the positive direction of the first axis). If the grid is
regular in lat-long, replace "x" and "y" in the previous sentence with
"long" and "lat," respectively.
:raises: ValueError: if extraction fails and raise_error_if_fails = True.
"""
# Error-checking.
error_checking.assert_is_string(field_name_grib1)
error_checking.assert_is_integer(num_grid_rows)
error_checking.assert_is_greater(num_grid_rows, 0)
error_checking.assert_is_integer(num_grid_columns)
error_checking.assert_is_greater(num_grid_columns, 0)
error_checking.assert_file_exists(wgrib_exe_name)
error_checking.assert_file_exists(wgrib2_exe_name)
error_checking.assert_is_boolean(raise_error_if_fails)
if sentinel_value is not None:
error_checking.assert_is_not_nan(sentinel_value)
# Housekeeping.
grib_file_type = file_name_to_type(grib_file_name)
if temporary_dir_name is not None:
file_system_utils.mkdir_recursive_if_necessary(
directory_name=temporary_dir_name)
temporary_file_name = tempfile.NamedTemporaryFile(dir=temporary_dir_name,
delete=False).name
# Extract field to temporary file.
if grib_file_type == GRIB1_FILE_TYPE:
command_string = (
'"{0:s}" "{1:s}" -s | grep -w "{2:s}" | "{0:s}" -i "{1:s}" '
'-text -nh -o "{3:s}"').format(wgrib_exe_name, grib_file_name,
field_name_grib1,
temporary_file_name)
else:
command_string = (
'"{0:s}" "{1:s}" -s | grep -w "{2:s}" | "{0:s}" -i "{1:s}" '
'-no_header -text "{3:s}"').format(
wgrib2_exe_name, grib_file_name,
_field_name_grib1_to_grib2(field_name_grib1),
temporary_file_name)
try:
subprocess.call(command_string, shell=True)
except OSError as this_exception:
os.remove(temporary_file_name)
if raise_error_if_fails:
raise
warning_string = (
'\n\n{0:s}\n\nCommand (shown above) failed (details shown below).'
'\n\n{1:s}').format(command_string, str(this_exception))
warnings.warn(warning_string)
return None
# Read field from temporary file.
field_vector = numpy.loadtxt(temporary_file_name)
os.remove(temporary_file_name)
try:
field_matrix = numpy.reshape(field_vector,
(num_grid_rows, num_grid_columns))
except ValueError as this_exception:
if raise_error_if_fails:
raise
warning_string = (
'\n\nnumpy.reshape failed (details shown below).\n\n{0:s}').format(
str(this_exception))
warnings.warn(warning_string)
return None
return _sentinel_value_to_nan(data_matrix=field_matrix,
sentinel_value=sentinel_value)
def write_pmm_file(pickle_file_name,
mean_denorm_input_matrices,
mean_denorm_output_matrices,
mean_initial_activation,
mean_final_activation,
model_file_name,
non_pmm_file_name,
pmm_max_percentile_level,
mean_sounding_pressures_pa=None):
"""Writes composite of backwards-optimized examples to Pickle file.
The composite should be created by probability-matched means (PMM).
T = number of input tensors to the model
H = number of sounding heights
:param pickle_file_name: Path to output file.
:param mean_denorm_input_matrices: See doc for `_check_in_and_out_matrices`.
:param mean_denorm_output_matrices: Same.
:param mean_initial_activation: Mean initial activation (before backwards
optimization).
:param mean_final_activation: Mean final activation (after backwards
optimization).
:param model_file_name: Path to model used for backwards optimization
(readable by `cnn.read_model`).
:param non_pmm_file_name: Path to standard backwards-optimization file
(containing non-composited results).
:param pmm_max_percentile_level: Max percentile level for PMM.
:param mean_sounding_pressures_pa: length-H numpy array of PMM-composited
sounding pressures. Needed only if `mean_denorm_input_matrices`
contains soundings from real examples but without pressure as a
predictor.
"""
error_checking.assert_is_string(model_file_name)
error_checking.assert_is_string(non_pmm_file_name)
error_checking.assert_is_geq(pmm_max_percentile_level, 90.)
error_checking.assert_is_leq(pmm_max_percentile_level, 100.)
_check_in_and_out_matrices(input_matrices=mean_denorm_input_matrices,
num_examples=None,
output_matrices=mean_denorm_output_matrices)
error_checking.assert_is_not_nan(mean_initial_activation)
error_checking.assert_is_not_nan(mean_final_activation)
if mean_sounding_pressures_pa is not None:
num_heights = mean_denorm_input_matrices[-1].shape[-2]
these_expected_dim = numpy.array([num_heights], dtype=int)
error_checking.assert_is_geq_numpy_array(mean_sounding_pressures_pa,
0.)
error_checking.assert_is_numpy_array(
mean_sounding_pressures_pa, exact_dimensions=these_expected_dim)
mean_bwo_dictionary = {
MEAN_INPUT_MATRICES_KEY: mean_denorm_input_matrices,
MEAN_OUTPUT_MATRICES_KEY: mean_denorm_output_matrices,
MEAN_INITIAL_ACTIVATION_KEY: mean_initial_activation,
MEAN_FINAL_ACTIVATION_KEY: mean_final_activation,
MODEL_FILE_KEY: model_file_name,
NON_PMM_FILE_KEY: non_pmm_file_name,
PMM_MAX_PERCENTILE_KEY: pmm_max_percentile_level,
MEAN_SOUNDING_PRESSURES_KEY: mean_sounding_pressures_pa
}
file_system_utils.mkdir_recursive_if_necessary(file_name=pickle_file_name)
pickle_file_handle = open(pickle_file_name, 'wb')
pickle.dump(mean_bwo_dictionary, pickle_file_handle)
pickle_file_handle.close()
def plot_attributes_diagram(figure_object,
axes_object,
mean_predictions,
mean_observations,
mean_value_in_training,
min_value_to_plot,
max_value_to_plot,
line_colour=RELIABILITY_LINE_COLOUR,
line_style='solid',
line_width=RELIABILITY_LINE_WIDTH,
example_counts=None,
inv_mean_observations=None,
inv_example_counts=None):
"""Plots attributes diagram.
If `example_counts is None`, will not plot histogram of predicted values.
If `inv_mean_observations is None` and `inv_example_counts is None`, will
not plot histogram of observed values.
B = number of bins
:param figure_object: Will plot on this figure (instance of
`matplotlib.figure.Figure`).
:param axes_object: Will plot on these axes (instance of
`matplotlib.axes._subplots.AxesSubplot`).
:param mean_predictions: length-B numpy array of mean predicted values.
:param mean_observations: length-B numpy array of mean observed values.
:param mean_value_in_training: Mean of target variable in training data.
:param min_value_to_plot: Minimum value in plot (for both x- and y-axes).
:param max_value_to_plot: Max value in plot (for both x- and y-axes).
If None, will be determined automatically.
:param line_colour: See doc for `_plot_reliability_curve`.
:param line_width: Same.
:param line_style: Same.
:param example_counts: length-B numpy array with number of examples in each
bin.
:param inv_mean_observations: length-B numpy array of mean observed values
for inverted reliability curve.
:param inv_example_counts: length-B numpy array of example counts for
inverted reliability curve.
:return: main_line_handle: See doc for `_plot_reliability_curve`.
"""
# Check input args.
error_checking.assert_is_numpy_array(mean_predictions, num_dimensions=1)
num_bins = len(mean_predictions)
expected_dim = numpy.array([num_bins], dtype=int)
error_checking.assert_is_numpy_array(mean_observations,
exact_dimensions=expected_dim)
plot_prediction_histogram = example_counts is not None
if plot_prediction_histogram:
error_checking.assert_is_integer_numpy_array(example_counts)
error_checking.assert_is_geq_numpy_array(example_counts, 0)
error_checking.assert_is_numpy_array(example_counts,
exact_dimensions=expected_dim)
error_checking.assert_is_not_nan(mean_value_in_training)
error_checking.assert_is_geq(max_value_to_plot, min_value_to_plot)
if max_value_to_plot == min_value_to_plot:
max_value_to_plot = min_value_to_plot + 1.
plot_obs_histogram = not (inv_mean_observations is None
and inv_example_counts is None)
if plot_obs_histogram:
error_checking.assert_is_numpy_array(inv_mean_observations,
exact_dimensions=expected_dim)
error_checking.assert_is_integer_numpy_array(inv_example_counts)
error_checking.assert_is_geq_numpy_array(inv_example_counts, 0)
error_checking.assert_is_numpy_array(inv_example_counts,
exact_dimensions=expected_dim)
_plot_attr_diagram_background(
axes_object=axes_object,
mean_value_in_training=mean_value_in_training,
min_value_in_plot=min_value_to_plot,
max_value_in_plot=max_value_to_plot)
if plot_prediction_histogram:
plot_inset_histogram(figure_object=figure_object,
bin_centers=mean_predictions,
bin_counts=example_counts,
has_predictions=True,
bar_colour=line_colour)
if plot_obs_histogram:
plot_inset_histogram(figure_object=figure_object,
bin_centers=inv_mean_observations,
bin_counts=inv_example_counts,
has_predictions=False,
bar_colour=line_colour)
return _plot_reliability_curve(axes_object=axes_object,
mean_predictions=mean_predictions,
mean_observations=mean_observations,
min_value_to_plot=min_value_to_plot,
max_value_to_plot=max_value_to_plot,
line_colour=line_colour,
line_style=line_style,
line_width=line_width)
def test_assert_is_not_nan_too_many_inputs(self):
"""Checks assert_is_not_nan when input is array of floats."""
with self.assertRaises(TypeError):
error_checking.assert_is_not_nan(FLOAT_NUMPY_ARRAY)
def test_assert_is_not_nan_float(self):
"""Checks assert_is_not_nan when input is float."""
error_checking.assert_is_not_nan(SINGLE_FLOAT)
def test_assert_is_not_nan_boolean(self):
"""Checks assert_is_not_nan when input is Boolean."""
with self.assertRaises(TypeError):
error_checking.assert_is_not_nan(SINGLE_BOOLEAN)
def buffer_simple_polygon(vertex_x_metres,
vertex_y_metres,
max_buffer_dist_metres,
min_buffer_dist_metres=numpy.nan,
preserve_angles=False):
"""Creates buffer around simple polygon.
V_0 = number of original vertices
V = number of final vertices
:param vertex_x_metres: numpy array (length V_0) with x-coordinates of
original vertices.
:param vertex_y_metres: numpy array (length V_0) with y-coordinates of
original vertices.
:param max_buffer_dist_metres: Max buffer distance.
:param min_buffer_dist_metres: Minimum buffer distance. If NaN, the buffer
will be inclusive (i.e., the original polygon will be included in the
buffer). Otherwise, the buffer will be exclusive (i.e., the buffer will
not include the original polygon). For example, if
`min_buffer_dist_metres` = NaN and `max_buffer_dist_metres` = 5, the
buffer will include the original polygon and an area of 5 metres outside
the original polygon. However, if `min_buffer_dist_metres` = 0 and
`max_buffer_dist_metres` = 5, the buffer will include only an area of 5
metres outside the original polygon. If `min_buffer_dist_metres` = 1
and `max_buffer_dist_metres` = 5, the buffer will include only an area
of 1-5 metres outside the original polygon.
:param preserve_angles: Boolean flag. If True, will preserve the angles of
all vertices in the original polygon, which means that distance will not
be strictly respected. If False, will preserve buffer distances, which
means that vertex angles will not be strictly respected. We highly
recommend keeping this as False (True only for unit tests).
:return: buffered_polygon_object: `shapely.geometry.Polygon` object.
"""
_check_vertex_arrays(vertex_x_metres, vertex_y_metres, allow_nan=False)
error_checking.assert_is_geq(min_buffer_dist_metres, 0., allow_nan=True)
error_checking.assert_is_not_nan(max_buffer_dist_metres)
if not numpy.isnan(min_buffer_dist_metres):
error_checking.assert_is_greater(max_buffer_dist_metres,
min_buffer_dist_metres)
error_checking.assert_is_boolean(preserve_angles)
if preserve_angles:
join_style = shapely.geometry.JOIN_STYLE.mitre
else:
join_style = shapely.geometry.JOIN_STYLE.round
orig_polygon_object = vertex_arrays_to_polygon_object(
vertex_x_metres, vertex_y_metres)
max_buffer_polygon_object = orig_polygon_object.buffer(
max_buffer_dist_metres, join_style=join_style)
if numpy.isnan(min_buffer_dist_metres):
return max_buffer_polygon_object
min_buffer_polygon_object = orig_polygon_object.buffer(
min_buffer_dist_metres, join_style=join_style)
min_buffer_vertex_dict = polygon_object_to_vertex_arrays(
min_buffer_polygon_object)
max_buffer_vertex_dict = polygon_object_to_vertex_arrays(
max_buffer_polygon_object)
return vertex_arrays_to_polygon_object(
max_buffer_vertex_dict[EXTERIOR_X_COLUMN],
max_buffer_vertex_dict[EXTERIOR_Y_COLUMN],
hole_x_coords_list=[min_buffer_vertex_dict[EXTERIOR_X_COLUMN]],
hole_y_coords_list=[min_buffer_vertex_dict[EXTERIOR_Y_COLUMN]])
def read_polygons_from_netcdf(netcdf_file_name,
metadata_dict=None,
spc_date_unix_sec=None,
tracking_start_time_unix_sec=None,
tracking_end_time_unix_sec=None,
raise_error_if_fails=True):
"""Reads storm polygons (outlines of storm cells) from NetCDF file.
P = number of grid points in storm cell (different for each storm cell)
V = number of vertices in storm polygon (different for each storm cell)
If file cannot be opened, returns None.
:param netcdf_file_name: Path to input file.
:param metadata_dict: Dictionary with metadata for NetCDF file, created by
`radar_io.read_metadata_from_raw_file`.
:param spc_date_unix_sec: SPC date;
:param tracking_start_time_unix_sec: Start time for tracking period. This
can be found by `get_start_end_times_for_spc_date`.
:param tracking_end_time_unix_sec: End time for tracking period. This can
be found by `get_start_end_times_for_spc_date`.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error.
:return: polygon_table: If file cannot be opened and raise_error_if_fails =
False, this is None. Otherwise, it is a pandas DataFrame with the
following columns.
polygon_table.storm_id: String ID for storm cell.
polygon_table.unix_time_sec: Time in Unix format.
polygon_table.spc_date_unix_sec: SPC date in Unix format.
polygon_table.tracking_start_time_unix_sec: Start time for tracking period.
polygon_table.tracking_end_time_unix_sec: End time for tracking period.
polygon_table.centroid_lat_deg: Latitude at centroid of storm cell (deg N).
polygon_table.centroid_lng_deg: Longitude at centroid of storm cell (deg E).
polygon_table.grid_point_latitudes_deg: length-P numpy array with latitudes
(deg N) of grid points in storm cell.
polygon_table.grid_point_longitudes_deg: length-P numpy array with
longitudes (deg E) of grid points in storm cell.
polygon_table.grid_point_rows: length-P numpy array with row indices (all
integers) of grid points in storm cell.
polygon_table.grid_point_columns: length-P numpy array with column indices
(all integers) of grid points in storm cell.
polygon_table.polygon_object_latlng: Instance of `shapely.geometry.Polygon`
with vertices in lat-long coordinates.
polygon_table.polygon_object_rowcol: Instance of `shapely.geometry.Polygon`
with vertices in row-column coordinates.
"""
error_checking.assert_file_exists(netcdf_file_name)
error_checking.assert_is_integer(spc_date_unix_sec)
error_checking.assert_is_not_nan(spc_date_unix_sec)
error_checking.assert_is_integer(tracking_start_time_unix_sec)
error_checking.assert_is_not_nan(tracking_start_time_unix_sec)
error_checking.assert_is_integer(tracking_end_time_unix_sec)
error_checking.assert_is_not_nan(tracking_end_time_unix_sec)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
storm_id_var_name = metadata_dict[radar_io.FIELD_NAME_COLUMN]
storm_id_var_name_orig = metadata_dict[radar_io.FIELD_NAME_COLUMN_ORIG]
num_values = len(netcdf_dataset.variables[radar_io.GRID_ROW_COLUMN_ORIG])
if num_values == 0:
sparse_grid_dict = {
radar_io.GRID_ROW_COLUMN: numpy.array([], dtype=int),
radar_io.GRID_COLUMN_COLUMN: numpy.array([], dtype=int),
radar_io.NUM_GRID_CELL_COLUMN: numpy.array([], dtype=int),
storm_id_var_name: numpy.array([], dtype=int)
}
else:
sparse_grid_dict = {
radar_io.GRID_ROW_COLUMN:
netcdf_dataset.variables[radar_io.GRID_ROW_COLUMN_ORIG][:],
radar_io.GRID_COLUMN_COLUMN:
netcdf_dataset.variables[radar_io.GRID_COLUMN_COLUMN_ORIG][:],
radar_io.NUM_GRID_CELL_COLUMN:
netcdf_dataset.variables[radar_io.NUM_GRID_CELL_COLUMN_ORIG][:],
storm_id_var_name:
netcdf_dataset.variables[storm_id_var_name_orig][:]
}
netcdf_dataset.close()
sparse_grid_table = pandas.DataFrame.from_dict(sparse_grid_dict)
numeric_storm_id_matrix, _, _ = (radar_s2f.sparse_to_full_grid(
sparse_grid_table, metadata_dict))
polygon_table = _storm_id_matrix_to_coord_lists(numeric_storm_id_matrix)
num_storms = len(polygon_table.index)
unix_times_sec = numpy.full(num_storms,
metadata_dict[radar_io.UNIX_TIME_COLUMN],
dtype=int)
spc_dates_unix_sec = numpy.full(num_storms, spc_date_unix_sec, dtype=int)
tracking_start_times_unix_sec = numpy.full(num_storms,
tracking_start_time_unix_sec,
dtype=int)
tracking_end_times_unix_sec = numpy.full(num_storms,
tracking_end_time_unix_sec,
dtype=int)
spc_date_string = time_conversion.time_to_spc_date_string(
spc_date_unix_sec)
storm_ids = _append_spc_date_to_storm_ids(
polygon_table[tracking_io.STORM_ID_COLUMN].values, spc_date_string)
simple_array = numpy.full(num_storms, numpy.nan)
object_array = numpy.full(num_storms, numpy.nan, dtype=object)
nested_array = polygon_table[[
tracking_io.STORM_ID_COLUMN, tracking_io.STORM_ID_COLUMN
]].values.tolist()
argument_dict = {
tracking_io.STORM_ID_COLUMN: storm_ids,
tracking_io.TIME_COLUMN: unix_times_sec,
tracking_io.SPC_DATE_COLUMN: spc_dates_unix_sec,
tracking_io.TRACKING_START_TIME_COLUMN: tracking_start_times_unix_sec,
tracking_io.TRACKING_END_TIME_COLUMN: tracking_end_times_unix_sec,
tracking_io.CENTROID_LAT_COLUMN: simple_array,
tracking_io.CENTROID_LNG_COLUMN: simple_array,
tracking_io.GRID_POINT_LAT_COLUMN: nested_array,
tracking_io.GRID_POINT_LNG_COLUMN: nested_array,
tracking_io.POLYGON_OBJECT_LATLNG_COLUMN: object_array,
tracking_io.POLYGON_OBJECT_ROWCOL_COLUMN: object_array
}
polygon_table = polygon_table.assign(**argument_dict)
for i in range(num_storms):
these_vertex_rows, these_vertex_columns = (
polygons.grid_points_in_poly_to_vertices(
polygon_table[tracking_io.GRID_POINT_ROW_COLUMN].values[i],
polygon_table[tracking_io.GRID_POINT_COLUMN_COLUMN].values[i]))
(polygon_table[tracking_io.GRID_POINT_ROW_COLUMN].values[i],
polygon_table[tracking_io.GRID_POINT_COLUMN_COLUMN].values[i]) = (
polygons.simple_polygon_to_grid_points(these_vertex_rows,
these_vertex_columns))
(polygon_table[tracking_io.GRID_POINT_LAT_COLUMN].values[i],
polygon_table[tracking_io.GRID_POINT_LNG_COLUMN].values[i]) = (
radar_io.rowcol_to_latlng(
polygon_table[tracking_io.GRID_POINT_ROW_COLUMN].values[i],
polygon_table[tracking_io.GRID_POINT_COLUMN_COLUMN].values[i],
nw_grid_point_lat_deg=metadata_dict[
radar_io.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_io.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_io.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_io.LNG_SPACING_COLUMN]))
these_vertex_lat_deg, these_vertex_lng_deg = radar_io.rowcol_to_latlng(
these_vertex_rows,
these_vertex_columns,
nw_grid_point_lat_deg=metadata_dict[
radar_io.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_io.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_io.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_io.LNG_SPACING_COLUMN])
(polygon_table[tracking_io.CENTROID_LAT_COLUMN].values[i],
polygon_table[tracking_io.CENTROID_LNG_COLUMN].values[i]) = (
polygons.get_latlng_centroid(these_vertex_lat_deg,
these_vertex_lng_deg))
polygon_table[tracking_io.POLYGON_OBJECT_ROWCOL_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(these_vertex_columns,
these_vertex_rows))
polygon_table[tracking_io.POLYGON_OBJECT_LATLNG_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(these_vertex_lng_deg,
these_vertex_lat_deg))
return polygon_table
def read_field_from_grib_file(grib_file_name,
grib1_field_name=None,
single_field_file_name=None,
wgrib_exe_name=WGRIB_EXE_NAME_DEFAULT,
wgrib2_exe_name=WGRIB2_EXE_NAME_DEFAULT,
num_grid_rows=None,
num_grid_columns=None,
sentinel_value=None,
delete_single_field_file=True,
raise_error_if_fails=True):
"""Reads single field from grib1 or grib2 file.
A "single field" is one variable at one time step and all grid cells.
:param grib_file_name: Path to input (grib1 or grib2) file.
:param grib1_field_name: Field name in grib1 format (example: 500-mb height
is "HGT:500 mb").
:param single_field_file_name: Single field will be extracted from grib file
to here.
:param wgrib_exe_name: Path to wgrib executable.
:param wgrib2_exe_name: Path to wgrib2 executable.
:param num_grid_rows: Number of rows in grid.
:param num_grid_columns: Number of columns in grid.
:param sentinel_value: Sentinel value, all occurrences of which will be
replaced with NaN.
:param delete_single_field_file: Boolean flag. If True, single-field file
will be deleted immediately upon reading.
:param raise_error_if_fails: Boolean flag. If read fails and
raise_error_if_fails = True, will raise an error.
:return: field_matrix: See documentation for _read_single_field_from_file.
"""
error_checking.assert_file_exists(grib_file_name)
error_checking.assert_is_string(grib1_field_name)
error_checking.assert_file_exists(wgrib_exe_name)
error_checking.assert_file_exists(wgrib2_exe_name)
error_checking.assert_is_integer(num_grid_rows)
error_checking.assert_is_greater(num_grid_rows, 0)
error_checking.assert_is_integer(num_grid_columns)
error_checking.assert_is_greater(num_grid_columns, 0)
if sentinel_value is not None:
error_checking.assert_is_not_nan(sentinel_value)
error_checking.assert_is_boolean(delete_single_field_file)
error_checking.assert_is_boolean(raise_error_if_fails)
success = _extract_single_field_to_file(
grib_file_name,
grib1_field_name=grib1_field_name,
output_file_name=single_field_file_name,
wgrib_exe_name=wgrib_exe_name,
wgrib2_exe_name=wgrib2_exe_name,
raise_error_if_fails=raise_error_if_fails)
if not success:
return None
field_matrix = _read_single_field_from_file(
single_field_file_name,
num_grid_rows=num_grid_rows,
num_grid_columns=num_grid_columns,
sentinel_value=sentinel_value,
raise_error_if_fails=raise_error_if_fails)
if delete_single_field_file and os.path.isfile(single_field_file_name):
os.remove(single_field_file_name)
return field_matrix
def test_assert_is_not_nan_integer(self):
"""Checks assert_is_not_nan when input is integer."""
error_checking.assert_is_not_nan(SINGLE_INTEGER)
def test_assert_is_not_nan_complex(self):
"""Checks assert_is_not_nan when input is complex."""
with self.assertRaises(TypeError):
error_checking.assert_is_not_nan(SINGLE_COMPLEX_NUMBER)
def test_assert_is_not_nan_nan(self):
"""Checks assert_is_not_nan when input is NaN."""
with self.assertRaises(ValueError):
error_checking.assert_is_not_nan(numpy.nan)
