def _find_unique_events(
event_latitudes_deg, event_longitudes_deg, event_times_unix_sec):
"""Finds unique events.
N = original number of events
n = number of unique events
:param event_latitudes_deg: length-N numpy array of latitudes (deg N).
:param event_longitudes_deg: length-N numpy array of longitudes (deg E).
:param event_times_unix_sec: length-N numpy array of times.
:return: unique_latitudes_deg: length-n numpy array of latitudes (deg N).
:return: unique_longitudes_deg: length-n numpy array of longitudes (deg E).
:return: unique_times_unix_sec: length-n numpy array of times.
"""
event_latitudes_deg = number_rounding.round_to_nearest(
event_latitudes_deg, LATLNG_TOLERANCE_DEG)
event_longitudes_deg = number_rounding.round_to_nearest(
event_longitudes_deg, LATLNG_TOLERANCE_DEG)
coord_matrix = numpy.transpose(numpy.vstack((
event_latitudes_deg, event_longitudes_deg, event_times_unix_sec)))
_, unique_indices = numpy.unique(coord_matrix, return_index=True, axis=0)
return (event_latitudes_deg[unique_indices],
event_longitudes_deg[unique_indices],
event_times_unix_sec[unique_indices])
def _check_regression_params(min_lead_time_sec=None,
max_lead_time_sec=None,
min_distance_metres=None,
max_distance_metres=None,
percentile_level=None):
"""Error-checks (and if necessary, rounds) parameters for regression labels.
t = time of a given storm object
:param min_lead_time_sec: Minimum lead time (wind time minus storm-object
time). Wind observations occurring before t + min_lead_time_sec are
ignored.
:param max_lead_time_sec: Maximum lead time (wind time minus storm-object
time). Wind observations occurring after t + max_lead_time_sec are
ignored.
:param min_distance_metres: Minimum distance between storm boundary and wind
observations. Wind observations nearer to the storm are ignored.
:param max_distance_metres: Maximum distance between storm boundary and wind
observations. Wind observations farther from the storm are ignored.
:param percentile_level: The label for each storm object will be the [q]th-
percentile speed of wind observations in the given time and distance
ranges, where q = percentile_level.
:return: parameter_dict: Dictionary with the following keys.
parameter_dict['min_lead_time_sec']: Same as input, but maybe rounded.
parameter_dict['max_lead_time_sec']: Same as input, but maybe rounded.
parameter_dict['min_distance_metres']: Same as input, but maybe rounded.
parameter_dict['max_distance_metres']: Same as input, but maybe rounded.
parameter_dict['percentile_level']: Same as input, but maybe rounded.
"""
error_checking.assert_is_integer(min_lead_time_sec)
error_checking.assert_is_geq(min_lead_time_sec, 0)
error_checking.assert_is_integer(max_lead_time_sec)
error_checking.assert_is_geq(max_lead_time_sec, min_lead_time_sec)
error_checking.assert_is_geq(min_distance_metres, 0.)
error_checking.assert_is_geq(max_distance_metres, min_distance_metres)
error_checking.assert_is_geq(percentile_level, 0.)
error_checking.assert_is_leq(percentile_level, 100.)
min_distance_metres = rounder.round_to_nearest(min_distance_metres,
DISTANCE_PRECISION_METRES)
max_distance_metres = rounder.round_to_nearest(max_distance_metres,
DISTANCE_PRECISION_METRES)
percentile_level = rounder.round_to_nearest(percentile_level,
PERCENTILE_LEVEL_PRECISION)
return {
MIN_LEAD_TIME_NAME: min_lead_time_sec,
MAX_LEAD_TIME_NAME: max_lead_time_sec,
MIN_DISTANCE_NAME: min_distance_metres,
MAX_DISTANCE_NAME: max_distance_metres,
PERCENTILE_LEVEL_NAME: percentile_level
}
def rowcol_to_latlng(grid_rows, grid_columns, nw_grid_point_lat_deg,
nw_grid_point_lng_deg, lat_spacing_deg, lng_spacing_deg):
"""Converts radar coordinates from row-column to lat-long.
P = number of input grid points
:param grid_rows: length-P numpy array with row indices of grid points
(increasing from north to south).
:param grid_columns: length-P numpy array with column indices of grid points
(increasing from west to east).
:param nw_grid_point_lat_deg: Latitude (deg N) of northwesternmost grid
point.
:param nw_grid_point_lng_deg: Longitude (deg E) of northwesternmost grid
point.
:param lat_spacing_deg: Spacing (deg N) between meridionally adjacent grid
points.
:param lng_spacing_deg: Spacing (deg E) between zonally adjacent grid
points.
:return: latitudes_deg: length-P numpy array with latitudes (deg N) of grid
points.
:return: longitudes_deg: length-P numpy array with longitudes (deg E) of
grid points.
"""
error_checking.assert_is_real_numpy_array(grid_rows)
error_checking.assert_is_geq_numpy_array(grid_rows, -0.5, allow_nan=True)
error_checking.assert_is_numpy_array(grid_rows, num_dimensions=1)
num_points = len(grid_rows)
error_checking.assert_is_real_numpy_array(grid_columns)
error_checking.assert_is_geq_numpy_array(grid_columns,
-0.5,
allow_nan=True)
error_checking.assert_is_numpy_array(grid_columns,
exact_dimensions=numpy.array(
[num_points]))
error_checking.assert_is_valid_latitude(nw_grid_point_lat_deg)
nw_grid_point_lng_deg = lng_conversion.convert_lng_positive_in_west(
nw_grid_point_lng_deg, allow_nan=False)
error_checking.assert_is_greater(lat_spacing_deg, 0.)
error_checking.assert_is_greater(lng_spacing_deg, 0.)
latitudes_deg = rounder.round_to_nearest(
nw_grid_point_lat_deg - lat_spacing_deg * grid_rows,
lat_spacing_deg / 2)
longitudes_deg = rounder.round_to_nearest(
nw_grid_point_lng_deg + lng_spacing_deg * grid_columns,
lng_spacing_deg / 2)
return latitudes_deg, lng_conversion.convert_lng_positive_in_west(
longitudes_deg, allow_nan=True)
def plot_parallels(basemap_object,
axes_object,
min_latitude_deg=None,
max_latitude_deg=None,
num_parallels=DEFAULT_NUM_PARALLELS,
line_width=DEFAULT_GRID_LINE_WIDTH,
line_colour=DEFAULT_GRID_LINE_COLOUR,
z_order=DEFAULT_GRID_LINE_Z_ORDER):
"""Plots parallels (grid lines for latitude).
If `min_latitude_deg` and `max_latitude_deg` are both None, this method will
take plotting limits from `basemap_object`.
:param basemap_object: See doc for `plot_countries`.
:param axes_object: Same.
:param min_latitude_deg: Minimum latitude for grid lines.
:param max_latitude_deg: Max latitude for grid lines.
:param num_parallels: Number of parallels.
:param line_width: See doc for `plot_countries`.
:param line_colour: Same.
:param z_order: Same.
"""
if min_latitude_deg is None or max_latitude_deg is None:
min_latitude_deg = basemap_object.llcrnrlat
max_latitude_deg = basemap_object.urcrnrlat
error_checking.assert_is_valid_latitude(min_latitude_deg)
error_checking.assert_is_valid_latitude(max_latitude_deg)
error_checking.assert_is_greater(max_latitude_deg, min_latitude_deg)
error_checking.assert_is_integer(num_parallels)
error_checking.assert_is_geq(num_parallels, 2)
parallel_spacing_deg = ((max_latitude_deg - min_latitude_deg) /
(num_parallels - 1))
if parallel_spacing_deg < 1.:
parallel_spacing_deg = number_rounding.round_to_nearest(
parallel_spacing_deg, 0.1)
else:
parallel_spacing_deg = numpy.round(parallel_spacing_deg)
min_latitude_deg = number_rounding.ceiling_to_nearest(
min_latitude_deg, parallel_spacing_deg)
max_latitude_deg = number_rounding.floor_to_nearest(
max_latitude_deg, parallel_spacing_deg)
num_parallels = 1 + int(
numpy.round(
(max_latitude_deg - min_latitude_deg) / parallel_spacing_deg))
latitudes_deg = numpy.linspace(min_latitude_deg,
max_latitude_deg,
num=num_parallels)
basemap_object.drawparallels(latitudes_deg,
color=colour_from_numpy_to_tuple(line_colour),
linewidth=line_width,
labels=[True, False, False, False],
ax=axes_object,
zorder=z_order)
def _check_statistic_params(statistic_names, percentile_levels):
"""Ensures that parameters of statistic are valid.
:param statistic_names: 1-D list with names of non-percentile-based
statistics.
:param percentile_levels: 1-D numpy array of percentile levels.
:return: percentile_levels: Same as input, but rounded to the nearest 0.1%.
:raises: ValueError: if any element of `statistic_names` is not in
`STATISTIC_NAMES`.
"""
error_checking.assert_is_string_list(statistic_names)
error_checking.assert_is_numpy_array(numpy.array(statistic_names),
num_dimensions=1)
error_checking.assert_is_numpy_array(percentile_levels, num_dimensions=1)
error_checking.assert_is_geq_numpy_array(percentile_levels, 0.)
error_checking.assert_is_leq_numpy_array(percentile_levels, 100.)
for this_name in statistic_names:
if this_name in STATISTIC_NAMES:
continue
error_string = ('\n\n' + str(STATISTIC_NAMES) +
'\n\nValid statistic names ' +
'(listed above) do not include the following: "' +
this_name + '"')
raise ValueError(error_string)
return numpy.unique(
rounder.round_to_nearest(percentile_levels,
PERCENTILE_LEVEL_PRECISION))
def test_round_to_nearest_array(self):
"""Ensures correct output from round_to_nearest with array input."""
nearest_values = number_rounding.round_to_nearest(
INPUT_VALUES, ROUNDING_BASE)
self.assertTrue(
numpy.allclose(nearest_values,
EXPECTED_NEAREST_VALUES,
atol=TOLERANCE))
def test_round_to_nearest_scalar(self):
"""Ensures correct output from round_to_nearest with scalar input."""
nearest_scalar = number_rounding.round_to_nearest(
INPUT_VALUE_SCALAR, ROUNDING_BASE)
self.assertTrue(
numpy.isclose(nearest_scalar,
EXPECTED_NEAREST_SCALAR,
atol=TOLERANCE))
def latlng_to_rowcol(latitudes_deg, longitudes_deg, nw_grid_point_lat_deg=None,
nw_grid_point_lng_deg=None, lat_spacing_deg=None,
lng_spacing_deg=None):
"""Converts radar coordinates from lat-long to row-column.
P = number of points
:param latitudes_deg: length-P numpy array of latitudes (deg N).
:param longitudes_deg: length-P numpy array of longitudes (deg E).
:param nw_grid_point_lat_deg: Latitude (deg N) of northwesternmost grid
point.
:param nw_grid_point_lng_deg: Longitude (deg E) of northwesternmost grid
point.
:param lat_spacing_deg: Spacing (deg N) between adjacent rows.
:param lng_spacing_deg: Spacing (deg E) between adjacent columns.
:return: grid_rows: length-P numpy array of row indices (increasing from
north to south).
:return: grid_columns: length-P numpy array of column indices (increasing
from west to east).
"""
error_checking.assert_is_valid_lat_numpy_array(latitudes_deg,
allow_nan=True)
error_checking.assert_is_numpy_array(latitudes_deg, num_dimensions=1)
num_points = len(latitudes_deg)
longitudes_deg = lng_conversion.convert_lng_positive_in_west(longitudes_deg,
allow_nan=True)
error_checking.assert_is_numpy_array(
longitudes_deg, exact_dimensions=numpy.array([num_points]))
error_checking.assert_is_valid_latitude(nw_grid_point_lat_deg)
nw_grid_point_lng_deg = lng_conversion.convert_lng_positive_in_west(
nw_grid_point_lng_deg, allow_nan=False)
error_checking.assert_is_greater(lat_spacing_deg, 0)
error_checking.assert_is_greater(lng_spacing_deg, 0)
grid_columns = rounder.round_to_nearest(
(longitudes_deg - nw_grid_point_lng_deg) / lng_spacing_deg, 0.5)
grid_rows = rounder.round_to_nearest(
(nw_grid_point_lat_deg - latitudes_deg) / lat_spacing_deg, 0.5)
return grid_rows, grid_columns
def _check_class_cutoffs(class_cutoffs_kt):
"""Error-checks (and if necessary, rounds) cutoffs for classification.
C = number of classes
c = C - 1 = number of class cutoffs
:param class_cutoffs_kt: length-c numpy array of class cutoffs in knots
(nautical miles per hour).
:return: class_cutoffs_kt: Same as input, except with only unique rounded
values.
"""
class_cutoffs_kt = rounder.round_to_nearest(class_cutoffs_kt,
CLASS_CUTOFF_PRECISION_KT)
class_cutoffs_kt, _, _ = classifn_utils.classification_cutoffs_to_ranges(
class_cutoffs_kt, non_negative_only=True)
return class_cutoffs_kt
def _update_frequency_dict(frequency_dict, new_data_matrix, rounding_base):
"""Updates measurement frequencies.
:param frequency_dict: Dictionary, where each key is a measurement and the
corresponding value is the number of times said measurement occurs.
:param new_data_matrix: numpy array with new values. Will be used to
update frequencies in `frequency_dict`.
:param rounding_base: Each value in `new_data_matrix` will be rounded to the
nearest multiple of this base.
:return: frequency_dict: Same as input, but with new frequencies.
"""
new_unique_values, new_counts = numpy.unique(
number_rounding.round_to_nearest(new_data_matrix, rounding_base),
return_counts=True)
for i in range(len(new_unique_values)):
if new_unique_values[i] in frequency_dict:
frequency_dict[new_unique_values[i]] += new_counts[i]
else:
frequency_dict[new_unique_values[i]] = new_counts[i]
return frequency_dict
def update_frequency_dict(frequency_dict, new_data_matrix, rounding_base):
"""Uses new data to update frequencies for min-max normalization.
:param frequency_dict: Dictionary, where each key is a unique measurement
and the corresponding value is num times the measurement occurs.
:param new_data_matrix: numpy array with new data.
:param rounding_base: Rounding base used to discretize continuous values
into unique values.
:return: frequency_dict: Same as input but with updated values.
"""
error_checking.assert_is_numpy_array_without_nan(new_data_matrix)
new_unique_values, new_counts = numpy.unique(
number_rounding.round_to_nearest(new_data_matrix, rounding_base),
return_counts=True)
for i in range(len(new_unique_values)):
if new_unique_values[i] in frequency_dict:
frequency_dict[new_unique_values[i]] += new_counts[i]
else:
frequency_dict[new_unique_values[i]] = new_counts[i]
return frequency_dict
def _find_nearest_storms(
storm_object_table=None,
wind_table=None,
max_time_before_storm_start_sec=None,
max_time_after_storm_end_sec=None,
max_linkage_dist_metres=None,
interp_time_spacing_sec=INTERP_TIME_SPACING_DEFAULT_SEC):
"""Finds nearest storm cell to each wind observation.
N = number of wind observations
:param storm_object_table: pandas DataFrame created by
_storm_objects_to_cells.
:param wind_table: pandas DataFrame created by _project_winds_latlng_to_xy.
:param max_time_before_storm_start_sec: Max time before beginning of storm
cell. If wind observation W occurs > max_time_before_storm_start_sec
before the first time in storm cell S, W cannot be linked to S.
:param max_time_after_storm_end_sec: Max time after end of storm cell. If
wind observation W occurs > max_time_after_storm_end_sec after the last
time in storm cell S, W cannot be linked to S.
:param max_linkage_dist_metres: Max linkage distance. If wind observation W
is > max_linkage_dist_metres from the nearest storm cell, W will not be
linked to a storm cell.
:param interp_time_spacing_sec: Discretization time for interpolation. If
interp_time_spacing_sec = 1, storms will be interpolated to each unique
wind time (since wind times are rounded to the nearest second). If
interp_time_spacing_sec = q, storms will be interpolated to each unique
multiple of q seconds among wind times.
:return: wind_to_storm_table: Same as input, but with additional columns
listed below.
wind_to_storm_table.nearest_storm_id: String ID for nearest storm cell. If
the wind observation is not linked to a storm cell, this will be None.
wind_to_storm_table.linkage_distance_metres: Distance to nearest storm cell.
If the wind observation is not linked to a storm cell, this will be NaN.
"""
rounded_times_unix_sec = rounder.round_to_nearest(
wind_table[raw_wind_io.TIME_COLUMN].values, interp_time_spacing_sec)
unique_rounded_times_unix_sec, wind_times_orig_to_unique_rounded = (
numpy.unique(rounded_times_unix_sec, return_inverse=True))
unique_rounded_time_strings = [
time_conversion.unix_sec_to_string(t, TIME_FORMAT_FOR_LOG_MESSAGES)
for t in unique_rounded_times_unix_sec
]
num_wind_observations = len(wind_table.index)
nearest_storm_ids = [None] * num_wind_observations
linkage_distances_metres = numpy.full(num_wind_observations, numpy.nan)
num_unique_times = len(unique_rounded_times_unix_sec)
for i in range(num_unique_times):
print 'Linking wind observations at ~{0:s} to storms...'.format(
unique_rounded_time_strings[i])
these_wind_rows = numpy.where(
wind_times_orig_to_unique_rounded == i)[0]
this_interp_vertex_table = _interp_storms_in_time(
storm_object_table,
query_time_unix_sec=unique_rounded_times_unix_sec[i],
max_time_before_start_sec=max_time_before_storm_start_sec,
max_time_after_end_sec=max_time_after_storm_end_sec)
these_nearest_storm_ids, these_link_distances_metres = (
_find_nearest_storms_at_one_time(
this_interp_vertex_table,
wind_x_coords_metres=wind_table[WIND_X_COLUMN].
values[these_wind_rows],
wind_y_coords_metres=wind_table[WIND_Y_COLUMN].
values[these_wind_rows],
max_linkage_dist_metres=max_linkage_dist_metres))
linkage_distances_metres[these_wind_rows] = these_link_distances_metres
for j in range(len(these_wind_rows)):
nearest_storm_ids[these_wind_rows[j]] = these_nearest_storm_ids[j]
argument_dict = {
NEAREST_STORM_ID_COLUMN: nearest_storm_ids,
LINKAGE_DISTANCE_COLUMN: linkage_distances_metres
}
return wind_table.assign(**argument_dict)
def _check_target_params(
min_lead_time_sec, max_lead_time_sec, min_link_distance_metres,
max_link_distance_metres, tornadogenesis_only=None, min_fujita_rating=0,
wind_speed_percentile_level=None, wind_speed_cutoffs_kt=None):
"""Error-checks parameters for target variable.
:param min_lead_time_sec: Minimum lead time. For a storm object at time t,
the target value will be based only on events occurring at times from
[t + `min_lead_time_sec`, t + `max_lead_time_sec`].
:param max_lead_time_sec: See above.
:param min_link_distance_metres: Minimum linkage distance. For each storm
cell S, target values will be based only on events occurring at
distances of `min_link_distance_metres`...`max_link_distance_metres`
outside the storm cell. If `min_link_distance_metres` = 0, events
inside the storm cell will also be permitted.
:param max_link_distance_metres: See above.
:param tornadogenesis_only: Boolean flag. If True, labels are for
tornadogenesis. If False, labels are for occurrence (pre-existing
tornado or genesis). If None, labels are for wind speed.
:param min_fujita_rating: [used if `tornadogenesis_only == True`]
Minimum Fujita rating (integer).
:param wind_speed_percentile_level: [used if `tornadogenesis_only == False`]
Percentile level for wind speed. For each storm object s, the target
value will be based on the [q]th percentile, where
q = `wind_speed_percentile_level`, of all wind speeds linked to s.
:param wind_speed_cutoffs_kt: [used if `tornadogenesis_only == False`]
1-D numpy array of cutoffs (knots) for wind-speed classification. The
lower bound of the first bin will always be 0 kt, and the upper bound of
the last bin will always be infinity, so these values do not need to be
included.
:return: target_param_dict: Dictionary with the following keys.
target_param_dict['min_link_distance_metres']: Rounded version of input.
target_param_dict['min_link_distance_metres']: Rounded version of input.
target_param_dict['min_fujita_rating']: See input.
target_param_dict['wind_speed_percentile_level']: Rounded version of input.
target_param_dict['wind_speed_cutoffs_kt']: Same as input, but including
0 kt for lower bound of first bin and infinity for upper bound of last
bin.
"""
error_checking.assert_is_integer(min_lead_time_sec)
error_checking.assert_is_geq(min_lead_time_sec, 0)
error_checking.assert_is_integer(max_lead_time_sec)
error_checking.assert_is_geq(max_lead_time_sec, min_lead_time_sec)
min_link_distance_metres = number_rounding.round_to_nearest(
min_link_distance_metres, DISTANCE_PRECISION_METRES
)
max_link_distance_metres = number_rounding.round_to_nearest(
max_link_distance_metres, DISTANCE_PRECISION_METRES
)
error_checking.assert_is_geq(min_link_distance_metres, 0.)
error_checking.assert_is_geq(
max_link_distance_metres, min_link_distance_metres
)
if tornadogenesis_only is not None:
error_checking.assert_is_boolean(tornadogenesis_only)
error_checking.assert_is_integer(min_fujita_rating)
error_checking.assert_is_geq(min_fujita_rating, 0)
error_checking.assert_is_leq(min_fujita_rating, 5)
return {
MIN_LINKAGE_DISTANCE_KEY: min_link_distance_metres,
MAX_LINKAGE_DISTANCE_KEY: max_link_distance_metres,
MIN_FUJITA_RATING_KEY: min_fujita_rating,
PERCENTILE_LEVEL_KEY: None,
WIND_SPEED_CUTOFFS_KEY: None
}
wind_speed_percentile_level = number_rounding.round_to_nearest(
wind_speed_percentile_level, PERCENTILE_LEVEL_PRECISION
)
error_checking.assert_is_geq(wind_speed_percentile_level, 0.)
error_checking.assert_is_leq(wind_speed_percentile_level, 100.)
if wind_speed_cutoffs_kt is not None:
wind_speed_cutoffs_kt = number_rounding.round_to_nearest(
wind_speed_cutoffs_kt, WIND_SPEED_PRECISION_KT
)
wind_speed_cutoffs_kt = classifn_utils.classification_cutoffs_to_ranges(
wind_speed_cutoffs_kt, non_negative_only=True
)[0]
return {
MIN_LINKAGE_DISTANCE_KEY: min_link_distance_metres,
MAX_LINKAGE_DISTANCE_KEY: max_link_distance_metres,
PERCENTILE_LEVEL_KEY: wind_speed_percentile_level,
WIND_SPEED_CUTOFFS_KEY: wind_speed_cutoffs_kt
}
def _run(tropical_example_dir_name, non_tropical_example_dir_name,
output_file_name):
"""Plots all sites wtih data.
This is effectively the main method.
:param tropical_example_dir_name: See documentation at top of file.
:param non_tropical_example_dir_name: Same.
:param output_file_name: Same.
"""
first_time_unix_sec = (
time_conversion.first_and_last_times_in_year(FIRST_YEAR)[0])
last_time_unix_sec = (
time_conversion.first_and_last_times_in_year(LAST_YEAR)[-1])
tropical_file_names = example_io.find_many_files(
directory_name=tropical_example_dir_name,
first_time_unix_sec=first_time_unix_sec,
last_time_unix_sec=last_time_unix_sec,
raise_error_if_all_missing=True,
raise_error_if_any_missing=False)
non_tropical_file_names = example_io.find_many_files(
directory_name=non_tropical_example_dir_name,
first_time_unix_sec=first_time_unix_sec,
last_time_unix_sec=last_time_unix_sec,
raise_error_if_all_missing=True,
raise_error_if_any_missing=False)
latitudes_deg_n = numpy.array([])
longitudes_deg_e = numpy.array([])
for this_file_name in tropical_file_names:
print('Reading data from: "{0:s}"...'.format(this_file_name))
this_example_dict = example_io.read_file(this_file_name)
these_latitudes_deg_n = example_utils.get_field_from_dict(
example_dict=this_example_dict,
field_name=example_utils.LATITUDE_NAME)
these_longitudes_deg_e = example_utils.get_field_from_dict(
example_dict=this_example_dict,
field_name=example_utils.LONGITUDE_NAME)
latitudes_deg_n = numpy.concatenate(
(latitudes_deg_n, these_latitudes_deg_n))
longitudes_deg_e = numpy.concatenate(
(longitudes_deg_e, these_longitudes_deg_e))
for this_file_name in non_tropical_file_names:
print('Reading data from: "{0:s}"...'.format(this_file_name))
this_example_dict = example_io.read_file(this_file_name)
these_latitudes_deg_n = example_utils.get_field_from_dict(
example_dict=this_example_dict,
field_name=example_utils.LATITUDE_NAME)
these_longitudes_deg_e = example_utils.get_field_from_dict(
example_dict=this_example_dict,
field_name=example_utils.LONGITUDE_NAME)
latitudes_deg_n = numpy.concatenate(
(latitudes_deg_n, these_latitudes_deg_n))
longitudes_deg_e = numpy.concatenate(
(longitudes_deg_e, these_longitudes_deg_e))
coord_matrix = numpy.transpose(
numpy.vstack((latitudes_deg_n, longitudes_deg_e)))
coord_matrix = number_rounding.round_to_nearest(coord_matrix,
LATLNG_TOLERANCE_DEG)
coord_matrix = numpy.unique(coord_matrix, axis=0)
latitudes_deg_n = coord_matrix[:, 0]
longitudes_deg_e = coord_matrix[:, 1]
figure_object, axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=MIN_PLOT_LATITUDE_DEG_N,
max_latitude_deg=MAX_PLOT_LATITUDE_DEG_N,
min_longitude_deg=MIN_PLOT_LONGITUDE_DEG_E,
max_longitude_deg=MAX_PLOT_LONGITUDE_DEG_E,
resolution_string='l'))
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR,
line_width=BORDER_WIDTH)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR,
line_width=BORDER_WIDTH)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS,
line_colour=GRID_LINE_COLOUR,
line_width=GRID_LINE_WIDTH,
font_size=FONT_SIZE)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS,
line_colour=GRID_LINE_COLOUR,
line_width=GRID_LINE_WIDTH,
font_size=FONT_SIZE)
arctic_indices = numpy.where(latitudes_deg_n >= 66.5)[0]
print(len(arctic_indices))
arctic_x_coords, arctic_y_coords = basemap_object(
longitudes_deg_e[arctic_indices], latitudes_deg_n[arctic_indices])
axes_object.plot(arctic_x_coords,
arctic_y_coords,
linestyle='None',
marker=MARKER_TYPE,
markersize=MARKER_SIZE,
markeredgewidth=0,
markerfacecolor=ARCTIC_COLOUR,
markeredgecolor=ARCTIC_COLOUR)
mid_latitude_indices = numpy.where(
numpy.logical_and(latitudes_deg_n >= 30., latitudes_deg_n < 66.5))[0]
print(len(mid_latitude_indices))
mid_latitude_x_coords, mid_latitude_y_coords = basemap_object(
longitudes_deg_e[mid_latitude_indices],
latitudes_deg_n[mid_latitude_indices])
axes_object.plot(mid_latitude_x_coords,
mid_latitude_y_coords,
linestyle='None',
marker=MARKER_TYPE,
markersize=MARKER_SIZE,
markeredgewidth=0,
markerfacecolor=MID_LATITUDE_COLOUR,
markeredgecolor=MID_LATITUDE_COLOUR)
tropical_indices = numpy.where(latitudes_deg_n < 30.)[0]
print(len(tropical_indices))
tropical_x_coords, tropical_y_coords = basemap_object(
longitudes_deg_e[tropical_indices], latitudes_deg_n[tropical_indices])
axes_object.plot(tropical_x_coords,
tropical_y_coords,
linestyle='None',
marker=MARKER_TYPE,
markersize=MARKER_SIZE,
markeredgewidth=0,
markerfacecolor=TROPICAL_COLOUR,
markeredgecolor=TROPICAL_COLOUR)
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
figure_object.savefig(output_file_name,
dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0,
bbox_inches='tight')
pyplot.close(figure_object)
def _run(input_file_name, top_output_dir_name):
"""Splits predictions by site (point location).
This is effectively the main method.
:param input_file_name: See documentation at top of file.
:param top_output_dir_name: Same.
:raises: ValueError: if any example cannot be assigned to a site.
"""
# Read data.
print('Reading data from: "{0:s}"...'.format(input_file_name))
prediction_dict = prediction_io.read_file(input_file_name)
example_metadata_dict = example_utils.parse_example_ids(
prediction_dict[prediction_io.EXAMPLE_IDS_KEY])
example_latitudes_deg_n = number_rounding.round_to_nearest(
example_metadata_dict[example_utils.LATITUDES_KEY],
LATLNG_TOLERANCE_DEG)
example_longitudes_deg_e = number_rounding.round_to_nearest(
example_metadata_dict[example_utils.LONGITUDES_KEY],
LATLNG_TOLERANCE_DEG)
example_longitudes_deg_e = lng_conversion.convert_lng_positive_in_west(
example_longitudes_deg_e)
num_examples = len(example_latitudes_deg_n)
example_written_flags = numpy.full(num_examples, False, dtype=bool)
site_names = list(SITE_NAME_TO_LATLNG.keys())
num_sites = len(site_names)
for j in range(num_sites):
this_site_latitude_deg_n = SITE_NAME_TO_LATLNG[site_names[j]][0]
this_site_longitude_deg_e = SITE_NAME_TO_LATLNG[site_names[j]][1]
these_indices = numpy.where(
numpy.logical_and(
numpy.absolute(example_latitudes_deg_n -
this_site_latitude_deg_n) <=
LATLNG_TOLERANCE_DEG,
numpy.absolute(example_longitudes_deg_e -
this_site_longitude_deg_e) <=
LATLNG_TOLERANCE_DEG))[0]
this_prediction_dict = prediction_io.subset_by_index(
prediction_dict=copy.deepcopy(prediction_dict),
desired_indices=these_indices)
this_output_file_name = '{0:s}/{1:s}/predictions.nc'.format(
top_output_dir_name, site_names[j])
print('Writing {0:d} examples to: "{1:s}"...'.format(
len(these_indices), this_output_file_name))
if len(these_indices) == 0:
continue
example_written_flags[these_indices] = True
prediction_io.write_file(
netcdf_file_name=this_output_file_name,
scalar_target_matrix=this_prediction_dict[
prediction_io.SCALAR_TARGETS_KEY],
vector_target_matrix=this_prediction_dict[
prediction_io.VECTOR_TARGETS_KEY],
scalar_prediction_matrix=this_prediction_dict[
prediction_io.SCALAR_PREDICTIONS_KEY],
vector_prediction_matrix=this_prediction_dict[
prediction_io.VECTOR_PREDICTIONS_KEY],
heights_m_agl=this_prediction_dict[prediction_io.HEIGHTS_KEY],
example_id_strings=this_prediction_dict[
prediction_io.EXAMPLE_IDS_KEY],
model_file_name=this_prediction_dict[prediction_io.MODEL_FILE_KEY])
if numpy.all(example_written_flags):
return
# bad_latitudes_deg_n = (
# example_latitudes_deg_n[example_written_flags == False]
# )
# bad_longitudes_deg_e = (
# example_longitudes_deg_e[example_written_flags == False]
# )
# bad_coord_matrix = numpy.transpose(numpy.vstack((
# bad_latitudes_deg_n, bad_longitudes_deg_e
# )))
# bad_coord_matrix = numpy.unique(bad_coord_matrix, axis=0)
# print(bad_coord_matrix)
error_string = (
'{0:d} of {1:d} examples could not be assigned to a site. This is a '
'BIG PROBLEM.').format(numpy.sum(example_written_flags == False),
num_examples)
raise ValueError(error_string)
def plot_meridians(basemap_object,
axes_object,
min_longitude_deg=None,
max_longitude_deg=None,
num_meridians=DEFAULT_NUM_MERIDIANS,
line_width=DEFAULT_GRID_LINE_WIDTH,
line_colour=DEFAULT_GRID_LINE_COLOUR,
z_order=DEFAULT_GRID_LINE_Z_ORDER):
"""Plots meridians (grid lines for longitude).
If `min_longitude_deg` and `max_longitude_deg` are both None, this method
will take plotting limits from `basemap_object`.
:param basemap_object: See doc for `plot_countries`.
:param axes_object: Same.
:param min_longitude_deg: Minimum longitude for grid lines.
:param max_longitude_deg: Max longitude for grid lines.
:param num_meridians: Number of meridians.
:param line_width: See doc for `plot_countries`.
:param line_colour: Same.
:param z_order: Same.
"""
if min_longitude_deg is None or max_longitude_deg is None:
min_longitude_deg = basemap_object.llcrnrlon
max_longitude_deg = basemap_object.urcrnrlon
min_longitude_deg = lng_conversion.convert_lng_positive_in_west(
min_longitude_deg)
max_longitude_deg = lng_conversion.convert_lng_positive_in_west(
max_longitude_deg)
error_checking.assert_is_greater(max_longitude_deg, min_longitude_deg)
error_checking.assert_is_integer(num_meridians)
error_checking.assert_is_geq(num_meridians, 2)
meridian_spacing_deg = ((max_longitude_deg - min_longitude_deg) /
(num_meridians - 1))
if meridian_spacing_deg < 1.:
meridian_spacing_deg = number_rounding.round_to_nearest(
meridian_spacing_deg, 0.1)
else:
meridian_spacing_deg = numpy.round(meridian_spacing_deg)
min_longitude_deg = number_rounding.ceiling_to_nearest(
min_longitude_deg, meridian_spacing_deg)
max_longitude_deg = number_rounding.floor_to_nearest(
max_longitude_deg, meridian_spacing_deg)
num_meridians = 1 + int(
numpy.round(
(max_longitude_deg - min_longitude_deg) / meridian_spacing_deg))
longitudes_deg = numpy.linspace(min_longitude_deg,
max_longitude_deg,
num=num_meridians)
basemap_object.drawmeridians(longitudes_deg,
color=colour_from_numpy_to_tuple(line_colour),
linewidth=line_width,
labels=[False, False, False, True],
ax=axes_object,
zorder=z_order)
def _plot_tornado_and_radar(top_myrorss_dir_name, radar_field_name,
radar_height_m_asl, spc_date_string, tornado_table,
tornado_row, output_file_name):
"""Plots one unlinked tornado with radar field.
:param top_myrorss_dir_name: See documentation at top of file.
:param radar_field_name: Same.
:param radar_height_m_asl: Same.
:param spc_date_string: SPC date for linkage file (format "yyyymmdd").
:param tornado_table: pandas DataFrame created by
`linkage._read_input_tornado_reports`.
:param tornado_row: Will plot only tornado in [j]th row of table, where j =
`tornado_row`.
:param output_file_name: Path to output file. Figure will be saved here.
"""
tornado_time_unix_sec = tornado_table[
linkage.EVENT_TIME_COLUMN].values[tornado_row]
radar_time_unix_sec = number_rounding.round_to_nearest(
tornado_time_unix_sec, RADAR_TIME_INTERVAL_SEC)
radar_spc_date_string = time_conversion.time_to_spc_date_string(
radar_time_unix_sec)
radar_file_name = myrorss_and_mrms_io.find_raw_file(
top_directory_name=top_myrorss_dir_name,
spc_date_string=radar_spc_date_string,
unix_time_sec=radar_time_unix_sec,
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_field_name,
height_m_asl=radar_height_m_asl,
raise_error_if_missing=spc_date_string == radar_spc_date_string)
if not os.path.isfile(radar_file_name):
first_radar_time_unix_sec = number_rounding.ceiling_to_nearest(
time_conversion.get_start_of_spc_date(spc_date_string),
RADAR_TIME_INTERVAL_SEC)
last_radar_time_unix_sec = number_rounding.floor_to_nearest(
time_conversion.get_end_of_spc_date(spc_date_string),
RADAR_TIME_INTERVAL_SEC)
radar_time_unix_sec = max(
[radar_time_unix_sec, first_radar_time_unix_sec])
radar_time_unix_sec = min(
[radar_time_unix_sec, last_radar_time_unix_sec])
radar_file_name = myrorss_and_mrms_io.find_raw_file(
top_directory_name=top_myrorss_dir_name,
spc_date_string=spc_date_string,
unix_time_sec=radar_time_unix_sec,
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_field_name,
height_m_asl=radar_height_m_asl,
raise_error_if_missing=True)
radar_metadata_dict = myrorss_and_mrms_io.read_metadata_from_raw_file(
netcdf_file_name=radar_file_name,
data_source=radar_utils.MYRORSS_SOURCE_ID)
sparse_grid_table = (myrorss_and_mrms_io.read_data_from_sparse_grid_file(
netcdf_file_name=radar_file_name,
field_name_orig=radar_metadata_dict[
myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG],
data_source=radar_utils.MYRORSS_SOURCE_ID,
sentinel_values=radar_metadata_dict[radar_utils.SENTINEL_VALUE_COLUMN])
)
radar_matrix, grid_point_latitudes_deg, grid_point_longitudes_deg = (
radar_s2f.sparse_to_full_grid(sparse_grid_table=sparse_grid_table,
metadata_dict=radar_metadata_dict))
radar_matrix = numpy.flip(radar_matrix, axis=0)
grid_point_latitudes_deg = grid_point_latitudes_deg[::-1]
axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=numpy.min(grid_point_latitudes_deg),
max_latitude_deg=numpy.max(grid_point_latitudes_deg),
min_longitude_deg=numpy.min(grid_point_longitudes_deg),
max_longitude_deg=numpy.max(grid_point_longitudes_deg),
resolution_string='i')[1:])
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS)
radar_plotting.plot_latlng_grid(
field_matrix=radar_matrix,
field_name=radar_field_name,
axes_object=axes_object,
min_grid_point_latitude_deg=numpy.min(grid_point_latitudes_deg),
min_grid_point_longitude_deg=numpy.min(grid_point_longitudes_deg),
latitude_spacing_deg=numpy.diff(grid_point_latitudes_deg[:2])[0],
longitude_spacing_deg=numpy.diff(grid_point_longitudes_deg[:2])[0])
tornado_latitude_deg = tornado_table[
linkage.EVENT_LATITUDE_COLUMN].values[tornado_row]
tornado_longitude_deg = tornado_table[
linkage.EVENT_LONGITUDE_COLUMN].values[tornado_row]
axes_object.plot(tornado_longitude_deg,
tornado_latitude_deg,
linestyle='None',
marker=TORNADO_MARKER_TYPE,
markersize=TORNADO_MARKER_SIZE,
markeredgewidth=TORNADO_MARKER_EDGE_WIDTH,
markerfacecolor=plotting_utils.colour_from_numpy_to_tuple(
TORNADO_MARKER_COLOUR),
markeredgecolor=plotting_utils.colour_from_numpy_to_tuple(
TORNADO_MARKER_COLOUR))
tornado_time_string = time_conversion.unix_sec_to_string(
tornado_time_unix_sec, TIME_FORMAT)
title_string = (
'Unlinked tornado at {0:s}, {1:.2f} deg N, {2:.2f} deg E').format(
tornado_time_string, tornado_latitude_deg, tornado_longitude_deg)
pyplot.title(title_string, fontsize=TITLE_FONT_SIZE)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
imagemagick_utils.trim_whitespace(input_file_name=output_file_name,
output_file_name=output_file_name)
def write_field_to_myrorss_file(field_matrix,
netcdf_file_name,
field_name,
metadata_dict,
height_m_asl=None):
"""Writes field to MYRORSS-formatted file.
M = number of rows (unique grid-point latitudes)
N = number of columns (unique grid-point longitudes)
:param field_matrix: M-by-N numpy array with one radar variable at one time.
Latitude should increase down each column, and longitude should increase
to the right along each row.
:param netcdf_file_name: Path to output file.
:param field_name: Name of radar field in GewitterGefahr format.
:param metadata_dict: Dictionary created by either
`gridrad_io.read_metadata_from_full_grid_file` or
`read_metadata_from_raw_file`.
:param height_m_asl: Height of radar field (metres above sea level).
"""
if field_name == radar_utils.REFL_NAME:
field_to_heights_dict_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=[field_name],
data_source=radar_utils.MYRORSS_SOURCE_ID,
refl_heights_m_asl=numpy.array([height_m_asl])))
else:
field_to_heights_dict_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=[field_name],
data_source=radar_utils.MYRORSS_SOURCE_ID))
field_name = list(field_to_heights_dict_m_asl.keys())[0]
radar_height_m_asl = field_to_heights_dict_m_asl[field_name][0]
if field_name in radar_utils.ECHO_TOP_NAMES:
field_matrix = METRES_TO_KM * field_matrix
field_name_myrorss = radar_utils.field_name_new_to_orig(
field_name=field_name, data_source_name=radar_utils.MYRORSS_SOURCE_ID)
file_system_utils.mkdir_recursive_if_necessary(file_name=netcdf_file_name)
netcdf_dataset = Dataset(netcdf_file_name,
'w',
format='NETCDF3_64BIT_OFFSET')
netcdf_dataset.setncattr(FIELD_NAME_COLUMN_ORIG, field_name_myrorss)
netcdf_dataset.setncattr('DataType', 'SparseLatLonGrid')
netcdf_dataset.setncattr(
NW_GRID_POINT_LAT_COLUMN_ORIG,
rounder.round_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(
NW_GRID_POINT_LNG_COLUMN_ORIG,
rounder.round_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LNG_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(HEIGHT_COLUMN_ORIG,
METRES_TO_KM * numpy.float(radar_height_m_asl))
netcdf_dataset.setncattr(
UNIX_TIME_COLUMN_ORIG,
numpy.int32(metadata_dict[radar_utils.UNIX_TIME_COLUMN]))
netcdf_dataset.setncattr('FractionalTime', 0.)
netcdf_dataset.setncattr('attributes', ' ColorMap SubType Unit')
netcdf_dataset.setncattr('ColorMap-unit', 'dimensionless')
netcdf_dataset.setncattr('ColorMap-value', '')
netcdf_dataset.setncattr('SubType-unit', 'dimensionless')
netcdf_dataset.setncattr('SubType-value', numpy.float(radar_height_m_asl))
netcdf_dataset.setncattr('Unit-unit', 'dimensionless')
netcdf_dataset.setncattr('Unit-value', 'dimensionless')
netcdf_dataset.setncattr(
LAT_SPACING_COLUMN_ORIG,
rounder.round_to_nearest(metadata_dict[radar_utils.LAT_SPACING_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(
LNG_SPACING_COLUMN_ORIG,
rounder.round_to_nearest(metadata_dict[radar_utils.LNG_SPACING_COLUMN],
LATLNG_MULTIPLE_DEG))
netcdf_dataset.setncattr(SENTINEL_VALUE_COLUMNS_ORIG[0],
numpy.double(-99000.))
netcdf_dataset.setncattr(SENTINEL_VALUE_COLUMNS_ORIG[1],
numpy.double(-99001.))
min_latitude_deg = metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN] - (
metadata_dict[radar_utils.LAT_SPACING_COLUMN] *
(metadata_dict[radar_utils.NUM_LAT_COLUMN] - 1))
unique_grid_point_lats_deg, unique_grid_point_lngs_deg = (
grids.get_latlng_grid_points(
min_latitude_deg=min_latitude_deg,
min_longitude_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN],
num_rows=metadata_dict[radar_utils.NUM_LAT_COLUMN],
num_columns=metadata_dict[radar_utils.NUM_LNG_COLUMN]))
num_grid_rows = len(unique_grid_point_lats_deg)
num_grid_columns = len(unique_grid_point_lngs_deg)
field_vector = numpy.reshape(field_matrix,
num_grid_rows * num_grid_columns)
grid_point_lat_matrix, grid_point_lng_matrix = (
grids.latlng_vectors_to_matrices(unique_grid_point_lats_deg,
unique_grid_point_lngs_deg))
grid_point_lat_vector = numpy.reshape(grid_point_lat_matrix,
num_grid_rows * num_grid_columns)
grid_point_lng_vector = numpy.reshape(grid_point_lng_matrix,
num_grid_rows * num_grid_columns)
real_value_indices = numpy.where(numpy.invert(
numpy.isnan(field_vector)))[0]
netcdf_dataset.createDimension(NUM_LAT_COLUMN_ORIG, num_grid_rows - 1)
netcdf_dataset.createDimension(NUM_LNG_COLUMN_ORIG, num_grid_columns - 1)
netcdf_dataset.createDimension(NUM_PIXELS_COLUMN_ORIG,
len(real_value_indices))
row_index_vector, column_index_vector = radar_utils.latlng_to_rowcol(
grid_point_lat_vector,
grid_point_lng_vector,
nw_grid_point_lat_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN])
netcdf_dataset.createVariable(field_name_myrorss, numpy.single,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.createVariable(GRID_ROW_COLUMN_ORIG, numpy.int16,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.createVariable(GRID_COLUMN_COLUMN_ORIG, numpy.int16,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.createVariable(NUM_GRID_CELL_COLUMN_ORIG, numpy.int32,
(NUM_PIXELS_COLUMN_ORIG, ))
netcdf_dataset.variables[field_name_myrorss].setncattr(
'BackgroundValue', numpy.int32(-99900))
netcdf_dataset.variables[field_name_myrorss].setncattr(
'units', 'dimensionless')
netcdf_dataset.variables[field_name_myrorss].setncattr(
'NumValidRuns', numpy.int32(len(real_value_indices)))
netcdf_dataset.variables[field_name_myrorss][:] = field_vector[
real_value_indices]
netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG][:] = (
row_index_vector[real_value_indices])
netcdf_dataset.variables[GRID_COLUMN_COLUMN_ORIG][:] = (
column_index_vector[real_value_indices])
netcdf_dataset.variables[NUM_GRID_CELL_COLUMN_ORIG][:] = (numpy.full(
len(real_value_indices), 1, dtype=int))
netcdf_dataset.close()
