def create_latlng_grid(min_latitude_deg, max_latitude_deg,
latitude_spacing_deg, min_longitude_deg,
max_longitude_deg, longitude_spacing_deg):
"""Creates lat-long grid.
M = number of rows in grid
N = number of columns in grid
:param min_latitude_deg: Minimum latitude (deg N) in grid.
:param max_latitude_deg: Max latitude (deg N) in grid.
:param latitude_spacing_deg: Spacing (deg N) between grid points in adjacent
rows.
:param min_longitude_deg: Minimum longitude (deg E) in grid.
:param max_longitude_deg: Max longitude (deg E) in grid.
:param longitude_spacing_deg: Spacing (deg E) between grid points in
adjacent columns.
:return: grid_point_latitudes_deg: length-M numpy array with latitudes
(deg N) of grid points.
:return: grid_point_longitudes_deg: length-N numpy array with longitudes
(deg E) of grid points.
"""
# TODO(thunderhoser): Make this handle wrap-around issues.
min_longitude_deg = longitude_conv.convert_lng_positive_in_west(
min_longitude_deg)
max_longitude_deg = longitude_conv.convert_lng_positive_in_west(
max_longitude_deg)
min_latitude_deg = number_rounding.floor_to_nearest(
min_latitude_deg, latitude_spacing_deg)
max_latitude_deg = number_rounding.ceiling_to_nearest(
max_latitude_deg, latitude_spacing_deg)
min_longitude_deg = number_rounding.floor_to_nearest(
min_longitude_deg, longitude_spacing_deg)
max_longitude_deg = number_rounding.ceiling_to_nearest(
max_longitude_deg, longitude_spacing_deg)
num_grid_rows = 1 + int(
numpy.round(
(max_latitude_deg - min_latitude_deg) / latitude_spacing_deg))
num_grid_columns = 1 + int(
numpy.round(
(max_longitude_deg - min_longitude_deg) / longitude_spacing_deg))
return grids.get_latlng_grid_points(min_latitude_deg=min_latitude_deg,
min_longitude_deg=min_longitude_deg,
lat_spacing_deg=latitude_spacing_deg,
lng_spacing_deg=longitude_spacing_deg,
num_rows=num_grid_rows,
num_columns=num_grid_columns)
def plot_parallels(basemap_object,
axes_object,
min_latitude_deg=None,
max_latitude_deg=None,
num_parallels=DEFAULT_NUM_PARALLELS,
line_width=DEFAULT_GRID_LINE_WIDTH,
line_colour=DEFAULT_GRID_LINE_COLOUR,
z_order=DEFAULT_GRID_LINE_Z_ORDER):
"""Plots parallels (grid lines for latitude).
If `min_latitude_deg` and `max_latitude_deg` are both None, this method will
take plotting limits from `basemap_object`.
:param basemap_object: See doc for `plot_countries`.
:param axes_object: Same.
:param min_latitude_deg: Minimum latitude for grid lines.
:param max_latitude_deg: Max latitude for grid lines.
:param num_parallels: Number of parallels.
:param line_width: See doc for `plot_countries`.
:param line_colour: Same.
:param z_order: Same.
"""
if min_latitude_deg is None or max_latitude_deg is None:
min_latitude_deg = basemap_object.llcrnrlat
max_latitude_deg = basemap_object.urcrnrlat
error_checking.assert_is_valid_latitude(min_latitude_deg)
error_checking.assert_is_valid_latitude(max_latitude_deg)
error_checking.assert_is_greater(max_latitude_deg, min_latitude_deg)
error_checking.assert_is_integer(num_parallels)
error_checking.assert_is_geq(num_parallels, 2)
parallel_spacing_deg = ((max_latitude_deg - min_latitude_deg) /
(num_parallels - 1))
if parallel_spacing_deg < 1.:
parallel_spacing_deg = number_rounding.round_to_nearest(
parallel_spacing_deg, 0.1)
else:
parallel_spacing_deg = numpy.round(parallel_spacing_deg)
min_latitude_deg = number_rounding.ceiling_to_nearest(
min_latitude_deg, parallel_spacing_deg)
max_latitude_deg = number_rounding.floor_to_nearest(
max_latitude_deg, parallel_spacing_deg)
num_parallels = 1 + int(
numpy.round(
(max_latitude_deg - min_latitude_deg) / parallel_spacing_deg))
latitudes_deg = numpy.linspace(min_latitude_deg,
max_latitude_deg,
num=num_parallels)
basemap_object.drawparallels(latitudes_deg,
color=colour_from_numpy_to_tuple(line_colour),
linewidth=line_width,
labels=[True, False, False, False],
ax=axes_object,
zorder=z_order)
def find_processed_hourly_files(start_time_unix_sec=None,
end_time_unix_sec=None,
primary_source=None,
secondary_source=None,
top_directory_name=None,
raise_error_if_missing=True):
"""Finds processed hourly wind files on local machine.
N = number of hours in time period (start_time_unix_sec...end_time_unix_sec)
:param start_time_unix_sec: Beginning of time period.
:param end_time_unix_sec: End of time period.
:param primary_source: String ID for primary data source.
:param secondary_source: String ID for secondary data source.
:param top_directory_name: Name of top-level directory with processed wind
files.
:param raise_error_if_missing: Boolean flag. If True and *any* file is
missing, this method will raise an error.
:return: processed_file_names: length-N list of paths to processed files.
:return: hours_unix_sec: length-N numpy array of corresponding hours.
"""
min_hour_unix_sec = int(
rounder.floor_to_nearest(start_time_unix_sec, HOURS_TO_SECONDS))
max_hour_unix_sec = int(
rounder.floor_to_nearest(end_time_unix_sec, HOURS_TO_SECONDS))
hours_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=min_hour_unix_sec,
end_time_unix_sec=max_hour_unix_sec,
time_interval_sec=HOURS_TO_SECONDS,
include_endpoint=True)
num_hours = len(hours_unix_sec)
processed_file_names = [''] * num_hours
for i in range(num_hours):
processed_file_names[i] = find_processed_file(
start_time_unix_sec=hours_unix_sec[i],
end_time_unix_sec=hours_unix_sec[i] + HOURS_TO_SECONDS - 1,
primary_source=primary_source,
secondary_source=secondary_source,
top_directory_name=top_directory_name,
raise_error_if_missing=raise_error_if_missing)
return processed_file_names, hours_unix_sec
def test_floor_to_nearest_array(self):
"""Ensures correct output from floor_to_nearest with array input."""
floor_values = number_rounding.floor_to_nearest(
INPUT_VALUES, ROUNDING_BASE)
self.assertTrue(
numpy.allclose(floor_values, EXPECTED_FLOOR_VALUES,
atol=TOLERANCE))
def _plot_inset_histogram_for_attributes_diagram(
figure_object,
num_examples_by_bin,
bar_face_colour=DEFAULT_HISTOGRAM_FACE_COLOUR,
bar_edge_colour=DEFAULT_HISTOGRAM_EDGE_COLOUR,
bar_edge_width=DEFAULT_HISTOGRAM_EDGE_WIDTH):
"""Plots forecast histogram inset in attributes diagram.
For more on the attributes diagram, see Hsu and Murphy (1986).
B = number of forecast bins
:param figure_object: Instance of `matplotlib.figure.Figure`.
:param num_examples_by_bin: length-B numpy array with number of examples in
each forecast bin.
:param bar_face_colour: Colour (in any format accepted by
`matplotlib.colors`) for interior of histogram bars.
:param bar_edge_colour: Colour for edge of histogram bars.
:param bar_edge_width: Width for edge of histogram bars.
"""
error_checking.assert_is_integer_numpy_array(num_examples_by_bin)
error_checking.assert_is_numpy_array(num_examples_by_bin, num_dimensions=1)
error_checking.assert_is_geq_numpy_array(num_examples_by_bin, 0)
num_forecast_bins = len(num_examples_by_bin)
error_checking.assert_is_geq(num_forecast_bins, 2)
example_frequency_by_bin = (num_examples_by_bin.astype(float) /
numpy.sum(num_examples_by_bin))
forecast_bin_edges = numpy.linspace(0., 1., num=num_forecast_bins + 1)
forecast_bin_width = forecast_bin_edges[1] - forecast_bin_edges[0]
forecast_bin_centers = forecast_bin_edges[:-1] + forecast_bin_width / 2
inset_axes_object = figure_object.add_axes([
INSET_HISTOGRAM_LEFT_EDGE, INSET_HISTOGRAM_BOTTOM_EDGE,
INSET_HISTOGRAM_WIDTH, INSET_HISTOGRAM_HEIGHT
])
inset_axes_object.bar(
forecast_bin_centers,
example_frequency_by_bin,
forecast_bin_width,
color=plotting_utils.colour_from_numpy_to_tuple(bar_face_colour),
edgecolor=plotting_utils.colour_from_numpy_to_tuple(bar_edge_colour),
linewidth=bar_edge_width)
max_y_tick_value = rounder.floor_to_nearest(
1.05 * numpy.max(example_frequency_by_bin),
INSET_HISTOGRAM_Y_TICK_SPACING)
num_y_ticks = 1 + int(
numpy.round(max_y_tick_value / INSET_HISTOGRAM_Y_TICK_SPACING))
y_tick_values = numpy.linspace(0., max_y_tick_value, num=num_y_ticks)
pyplot.xticks(INSET_HISTOGRAM_X_TICKS, axes=inset_axes_object)
pyplot.yticks(y_tick_values, axes=inset_axes_object)
inset_axes_object.set_xlim(0., 1.)
inset_axes_object.set_ylim(0., 1.05 * numpy.max(example_frequency_by_bin))
def test_floor_to_nearest_scalar(self):
"""Ensures correct output from floor_to_nearest with scalar input."""
floor_of_scalar = number_rounding.floor_to_nearest(
INPUT_VALUE_SCALAR, ROUNDING_BASE)
self.assertTrue(
numpy.isclose(floor_of_scalar,
EXPECTED_FLOOR_OF_SCALAR,
atol=TOLERANCE))
def time_to_spc_date_unix_sec(unix_time_sec):
"""Converts time to SPC date (both in Unix format).
:param unix_time_sec: Time in Unix format.
:return: spc_date_unix_sec: SPC date in Unix format. If the SPC date is
"Oct 28 2017" (120000 UTC 28 Oct - 115959 UTC 29 Oct 2017),
spc_date_unix_sec will be 180000 UTC 28 Oct 2017. In general,
spc_date_unix_sec will be 6 hours into the SPC date.
"""
error_checking.assert_is_integer(unix_time_sec)
return int(SECONDS_INTO_SPC_DATE_DEFAULT + rounder.floor_to_nearest(
unix_time_sec - DAYS_TO_SECONDS / 2, DAYS_TO_SECONDS))
def _plot_inset_histogram_for_attributes_diagram(figure_object,
num_examples_by_bin):
"""Plots forecast histogram inset in attributes diagram.
For more on the attributes diagram, see Hsu and Murphy (1986).
B = number of forecast bins
:param figure_object: Instance of `matplotlib.figure.Figure`.
:param num_examples_by_bin: length-B numpy array with number of examples in
each forecast bin.
"""
error_checking.assert_is_integer_numpy_array(num_examples_by_bin)
error_checking.assert_is_numpy_array(num_examples_by_bin, num_dimensions=1)
error_checking.assert_is_geq_numpy_array(num_examples_by_bin, 0)
num_forecast_bins = len(num_examples_by_bin)
error_checking.assert_is_geq(num_forecast_bins, 2)
example_frequency_by_bin = (num_examples_by_bin.astype(float) /
numpy.sum(num_examples_by_bin))
forecast_bin_edges = numpy.linspace(0., 1., num=num_forecast_bins + 1)
forecast_bin_width = forecast_bin_edges[1] - forecast_bin_edges[0]
forecast_bin_centers = forecast_bin_edges[:-1] + forecast_bin_width / 2
inset_axes_object = figure_object.add_axes([
HISTOGRAM_LEFT_EDGE, HISTOGRAM_BOTTOM_EDGE, HISTOGRAM_AXES_WIDTH,
HISTOGRAM_AXES_HEIGHT
])
inset_axes_object.bar(
forecast_bin_centers,
example_frequency_by_bin,
forecast_bin_width,
color=plotting_utils.colour_from_numpy_to_tuple(BAR_FACE_COLOUR),
edgecolor=plotting_utils.colour_from_numpy_to_tuple(BAR_EDGE_COLOUR),
linewidth=BAR_EDGE_WIDTH)
max_y_tick_value = rounder.floor_to_nearest(
1.05 * numpy.max(example_frequency_by_bin), HISTOGRAM_Y_SPACING)
num_y_ticks = 1 + int(numpy.round(max_y_tick_value / HISTOGRAM_Y_SPACING))
y_tick_values = numpy.linspace(0., max_y_tick_value, num=num_y_ticks)
pyplot.xticks(HISTOGRAM_X_VALUES, axes=inset_axes_object)
pyplot.yticks(y_tick_values, axes=inset_axes_object)
inset_axes_object.set_xlim(0., 1.)
inset_axes_object.set_ylim(0., 1.05 * numpy.max(example_frequency_by_bin))
inset_axes_object.set_title('Forecast histogram', fontsize=20)
def time_and_period_length_to_range(unix_time_sec, period_length_sec):
"""Converts single time and period length to range (start/end of period).
:param unix_time_sec: Single time (Unix format).
:param period_length_sec: Length of time period (seconds).
:return: start_time_unix_sec: Beginning of time period (Unix format).
:return: end_time_unix_sec: End of time period (Unix format).
"""
error_checking.assert_is_integer(unix_time_sec)
error_checking.assert_is_not_nan(unix_time_sec)
error_checking.assert_is_integer(period_length_sec)
start_time_unix_sec = int(rounder.floor_to_nearest(
float(unix_time_sec), period_length_sec))
return start_time_unix_sec, start_time_unix_sec + period_length_sec
def range_and_interval_to_list(start_time_unix_sec=None,
end_time_unix_sec=None,
time_interval_sec=None,
include_endpoint=True):
"""Converts time period from range and interval to list of exact times.
N = number of exact times
:param start_time_unix_sec: Start time (Unix format).
:param end_time_unix_sec: End time (Unix format).
:param time_interval_sec: Interval (seconds) between successive exact times.
:param include_endpoint: Boolean flag. If True, endpoint will be included
in list of time steps. If False, endpoint will be excluded.
:return: unix_times_sec: length-N numpy array of exact times (Unix format).
"""
error_checking.assert_is_integer(start_time_unix_sec)
error_checking.assert_is_not_nan(start_time_unix_sec)
error_checking.assert_is_integer(end_time_unix_sec)
error_checking.assert_is_not_nan(end_time_unix_sec)
error_checking.assert_is_integer(time_interval_sec)
error_checking.assert_is_boolean(include_endpoint)
if include_endpoint:
error_checking.assert_is_geq(end_time_unix_sec, start_time_unix_sec)
else:
error_checking.assert_is_greater(end_time_unix_sec,
start_time_unix_sec)
start_time_unix_sec = int(
rounder.floor_to_nearest(float(start_time_unix_sec),
time_interval_sec))
end_time_unix_sec = int(
rounder.ceiling_to_nearest(float(end_time_unix_sec),
time_interval_sec))
if not include_endpoint:
end_time_unix_sec -= time_interval_sec
num_time_steps = 1 + int(
numpy.round(
(end_time_unix_sec - start_time_unix_sec) / time_interval_sec))
return numpy.linspace(start_time_unix_sec,
end_time_unix_sec,
num=num_time_steps,
dtype=int)
def plot_meridians(basemap_object=None,
axes_object=None,
bottom_left_lng_deg=None,
upper_right_lng_deg=None,
meridian_spacing_deg=DEFAULT_MERIDIAN_SPACING_DEG,
line_width=DEFAULT_PARALLEL_MERIDIAN_WIDTH,
line_colour=DEFAULT_PARALLEL_MERIDIAN_COLOUR):
"""Draws meridians (lines of equal longitude).
:param basemap_object: Instance of `mpl_toolkits.basemap.Basemap`.
:param axes_object: Instance of `matplotlib.axes._subplots.AxesSubplot`.
:param bottom_left_lng_deg: Longitude at bottom-left corner (deg E).
:param upper_right_lng_deg: Longitude at upper-right corner (deg E).
:param meridian_spacing_deg: Spacing between successive meridians (deg N).
:param line_width: Line width (real positive number).
:param line_colour: Colour (in any format accepted by `matplotlib.colors`).
"""
bottom_left_lng_deg = lng_conversion.convert_lng_positive_in_west(
bottom_left_lng_deg)
upper_right_lng_deg = lng_conversion.convert_lng_positive_in_west(
upper_right_lng_deg)
error_checking.assert_is_greater(upper_right_lng_deg, bottom_left_lng_deg)
error_checking.assert_is_greater(meridian_spacing_deg, 0)
min_meridian_deg = rounder.ceiling_to_nearest(bottom_left_lng_deg,
meridian_spacing_deg)
max_meridian_deg = rounder.floor_to_nearest(upper_right_lng_deg,
meridian_spacing_deg)
num_meridians_deg = int(1 + (max_meridian_deg - min_meridian_deg) /
meridian_spacing_deg)
meridians_deg = numpy.linspace(min_meridian_deg,
max_meridian_deg,
num=num_meridians_deg)
basemap_object.drawmeridians(meridians_deg,
color=line_colour,
linewidth=line_width,
labels=[False, False, False, True],
ax=axes_object,
zorder=Z_ORDER_MERIDIANS_AND_PARALLELS)
def plot_parallels(basemap_object=None,
axes_object=None,
bottom_left_lat_deg=None,
upper_right_lat_deg=None,
parallel_spacing_deg=DEFAULT_PARALLEL_SPACING_DEG,
line_width=DEFAULT_PARALLEL_MERIDIAN_WIDTH,
line_colour=DEFAULT_PARALLEL_MERIDIAN_COLOUR):
"""Draws parallels (lines of equal latitude).
:param basemap_object: Instance of `mpl_toolkits.basemap.Basemap`.
:param axes_object: Instance of `matplotlib.axes._subplots.AxesSubplot`.
:param bottom_left_lat_deg: Latitude at bottom-left corner (deg N).
:param upper_right_lat_deg: Latitude at upper-right corner (deg N).
:param parallel_spacing_deg: Spacing between successive parallels (deg N).
:param line_width: Line width (real positive number).
:param line_colour: Colour (in any format accepted by `matplotlib.colors`).
"""
error_checking.assert_is_valid_latitude(bottom_left_lat_deg)
error_checking.assert_is_valid_latitude(upper_right_lat_deg)
error_checking.assert_is_greater(upper_right_lat_deg, bottom_left_lat_deg)
error_checking.assert_is_greater(parallel_spacing_deg, 0)
min_parallel_deg = rounder.ceiling_to_nearest(bottom_left_lat_deg,
parallel_spacing_deg)
max_parallel_deg = rounder.floor_to_nearest(upper_right_lat_deg,
parallel_spacing_deg)
num_parallels_deg = int(1 + (max_parallel_deg - min_parallel_deg) /
parallel_spacing_deg)
parallels_deg = numpy.linspace(min_parallel_deg,
max_parallel_deg,
num=num_parallels_deg)
basemap_object.drawparallels(parallels_deg,
color=line_colour,
linewidth=line_width,
labels=[True, False, False, False],
ax=axes_object,
zorder=Z_ORDER_MERIDIANS_AND_PARALLELS)
def _interp_soundings(spc_date_string, lead_times_seconds,
lag_time_for_convective_contamination_sec,
top_ruc_directory_name, top_rap_directory_name,
top_tracking_dir_name, tracking_scale_metres2,
top_output_dir_name):
"""Interpolates NWP sounding to each storm object at each lead time.
:param spc_date_string: See documentation at top of file.
:param lead_times_seconds: Same.
:param lag_time_for_convective_contamination_sec: Same.
:param top_ruc_directory_name: Same.
:param top_rap_directory_name: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param top_output_dir_name: Same.
:raises: ValueError: if model-initialization times needed are on opposite
sides of 0000 UTC 1 May 2012 (the cutoff between RUC and RAP models).
"""
lead_times_seconds = numpy.array(lead_times_seconds, dtype=int)
tracking_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_string,
source_name=tracking_utils.SEGMOTION_NAME,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
storm_object_table = tracking_io.read_many_files(tracking_file_names)
print(SEPARATOR_STRING)
first_storm_time_unix_sec = numpy.min(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
last_storm_time_unix_sec = numpy.max(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
first_init_time_unix_sec = number_rounding.floor_to_nearest(
(first_storm_time_unix_sec + numpy.min(lead_times_seconds) -
lag_time_for_convective_contamination_sec), HOURS_TO_SECONDS)
last_init_time_unix_sec = number_rounding.floor_to_nearest(
(last_storm_time_unix_sec + numpy.max(lead_times_seconds) -
lag_time_for_convective_contamination_sec), HOURS_TO_SECONDS)
extreme_init_times_unix_sec = numpy.array(
[first_init_time_unix_sec, last_init_time_unix_sec], dtype=int)
if numpy.all(extreme_init_times_unix_sec < FIRST_RAP_TIME_UNIX_SEC):
top_grib_directory_name = top_ruc_directory_name
model_name = nwp_model_utils.RUC_MODEL_NAME
elif numpy.all(extreme_init_times_unix_sec >= FIRST_RAP_TIME_UNIX_SEC):
top_grib_directory_name = top_rap_directory_name
model_name = nwp_model_utils.RAP_MODEL_NAME
else:
first_storm_time_string = time_conversion.unix_sec_to_string(
first_storm_time_unix_sec, STORM_TIME_FORMAT)
last_storm_time_string = time_conversion.unix_sec_to_string(
last_storm_time_unix_sec, STORM_TIME_FORMAT)
first_init_time_string = time_conversion.unix_sec_to_string(
first_init_time_unix_sec, MODEL_INIT_TIME_FORMAT)
last_init_time_string = time_conversion.unix_sec_to_string(
last_init_time_unix_sec, MODEL_INIT_TIME_FORMAT)
error_string = (
'First and last storm times are {0:s} and {1:s}. Thus, first and '
'last model-initialization times needed are {2:s} and {3:s}, which '
'are on opposite sides of {4:s} (the cutoff between RUC and RAP '
'models). The code is not generalized enough to interp data from '
'two different models. Sorry, eh?').format(
first_storm_time_string, last_storm_time_string,
first_init_time_string, last_init_time_string,
FIRST_RAP_TIME_STRING)
raise ValueError(error_string)
sounding_dict_by_lead_time = soundings.interp_soundings_to_storm_objects(
storm_object_table=storm_object_table,
top_grib_directory_name=top_grib_directory_name,
model_name=model_name,
use_all_grids=True,
height_levels_m_agl=soundings.DEFAULT_HEIGHT_LEVELS_M_AGL,
lead_times_seconds=lead_times_seconds,
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec,
wgrib_exe_name=WGRIB_EXE_NAME,
wgrib2_exe_name=WGRIB2_EXE_NAME,
raise_error_if_missing=False)
print(SEPARATOR_STRING)
num_lead_times = len(lead_times_seconds)
for k in range(num_lead_times):
this_sounding_file_name = soundings.find_sounding_file(
top_directory_name=top_output_dir_name,
spc_date_string=spc_date_string,
lead_time_seconds=lead_times_seconds[k],
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec,
raise_error_if_missing=False)
print(
'Writing soundings to: "{0:s}"...'.format(this_sounding_file_name))
soundings.write_soundings(
netcdf_file_name=this_sounding_file_name,
sounding_dict_height_coords=sounding_dict_by_lead_time[k],
lead_time_seconds=lead_times_seconds[k],
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec)
def plot_meridians(basemap_object,
axes_object,
min_longitude_deg=None,
max_longitude_deg=None,
num_meridians=DEFAULT_NUM_MERIDIANS,
line_width=DEFAULT_GRID_LINE_WIDTH,
line_colour=DEFAULT_GRID_LINE_COLOUR,
z_order=DEFAULT_GRID_LINE_Z_ORDER):
"""Plots meridians (grid lines for longitude).
If `min_longitude_deg` and `max_longitude_deg` are both None, this method
will take plotting limits from `basemap_object`.
:param basemap_object: See doc for `plot_countries`.
:param axes_object: Same.
:param min_longitude_deg: Minimum longitude for grid lines.
:param max_longitude_deg: Max longitude for grid lines.
:param num_meridians: Number of meridians.
:param line_width: See doc for `plot_countries`.
:param line_colour: Same.
:param z_order: Same.
"""
if min_longitude_deg is None or max_longitude_deg is None:
min_longitude_deg = basemap_object.llcrnrlon
max_longitude_deg = basemap_object.urcrnrlon
min_longitude_deg = lng_conversion.convert_lng_positive_in_west(
min_longitude_deg)
max_longitude_deg = lng_conversion.convert_lng_positive_in_west(
max_longitude_deg)
error_checking.assert_is_greater(max_longitude_deg, min_longitude_deg)
error_checking.assert_is_integer(num_meridians)
error_checking.assert_is_geq(num_meridians, 2)
meridian_spacing_deg = ((max_longitude_deg - min_longitude_deg) /
(num_meridians - 1))
if meridian_spacing_deg < 1.:
meridian_spacing_deg = number_rounding.round_to_nearest(
meridian_spacing_deg, 0.1)
else:
meridian_spacing_deg = numpy.round(meridian_spacing_deg)
min_longitude_deg = number_rounding.ceiling_to_nearest(
min_longitude_deg, meridian_spacing_deg)
max_longitude_deg = number_rounding.floor_to_nearest(
max_longitude_deg, meridian_spacing_deg)
num_meridians = 1 + int(
numpy.round(
(max_longitude_deg - min_longitude_deg) / meridian_spacing_deg))
longitudes_deg = numpy.linspace(min_longitude_deg,
max_longitude_deg,
num=num_meridians)
basemap_object.drawmeridians(longitudes_deg,
color=colour_from_numpy_to_tuple(line_colour),
linewidth=line_width,
labels=[False, False, False, True],
ax=axes_object,
zorder=z_order)
def read_metadata_from_raw_file(netcdf_file_name,
data_source,
raise_error_if_fails=True):
"""Reads metadata from raw (either MYRORSS or MRMS) file.
This file should contain one radar field at one height and valid time.
:param netcdf_file_name: Path to input file.
:param data_source: Data source (string).
:param raise_error_if_fails: Boolean flag. If True and file cannot be read,
this method will raise an error. If False and file cannot be read, will
return None.
:return: metadata_dict: Dictionary with the following keys.
metadata_dict['nw_grid_point_lat_deg']: Latitude (deg N) of northwesternmost
grid point.
metadata_dict['nw_grid_point_lng_deg']: Longitude (deg E) of
northwesternmost grid point.
metadata_dict['lat_spacing_deg']: Spacing (deg N) between meridionally
adjacent grid points.
metadata_dict['lng_spacing_deg']: Spacing (deg E) between zonally adjacent
grid points.
metadata_dict['num_lat_in_grid']: Number of rows (unique grid-point
latitudes).
metadata_dict['num_lng_in_grid']: Number of columns (unique grid-point
longitudes).
metadata_dict['height_m_asl']: Radar height (metres above ground level).
metadata_dict['unix_time_sec']: Valid time.
metadata_dict['field_name']: Name of radar field in GewitterGefahr format.
metadata_dict['field_name_orig']: Name of radar field in original (either
MYRORSS or MRMS) format.
metadata_dict['sentinel_values']: 1-D numpy array of sentinel values.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
field_name_orig = str(getattr(netcdf_dataset, FIELD_NAME_COLUMN_ORIG))
metadata_dict = {
radar_utils.NW_GRID_POINT_LAT_COLUMN:
getattr(netcdf_dataset, NW_GRID_POINT_LAT_COLUMN_ORIG),
radar_utils.NW_GRID_POINT_LNG_COLUMN:
lng_conversion.convert_lng_positive_in_west(getattr(
netcdf_dataset, NW_GRID_POINT_LNG_COLUMN_ORIG),
allow_nan=False),
radar_utils.LAT_SPACING_COLUMN:
getattr(netcdf_dataset, LAT_SPACING_COLUMN_ORIG),
radar_utils.LNG_SPACING_COLUMN:
getattr(netcdf_dataset, LNG_SPACING_COLUMN_ORIG),
radar_utils.NUM_LAT_COLUMN:
netcdf_dataset.dimensions[NUM_LAT_COLUMN_ORIG].size + 1,
radar_utils.NUM_LNG_COLUMN:
netcdf_dataset.dimensions[NUM_LNG_COLUMN_ORIG].size + 1,
radar_utils.HEIGHT_COLUMN:
getattr(netcdf_dataset, HEIGHT_COLUMN_ORIG),
radar_utils.UNIX_TIME_COLUMN:
getattr(netcdf_dataset, UNIX_TIME_COLUMN_ORIG),
FIELD_NAME_COLUMN_ORIG:
field_name_orig,
radar_utils.FIELD_NAME_COLUMN:
radar_utils.field_name_orig_to_new(field_name_orig=field_name_orig,
data_source_name=data_source)
}
latitude_spacing_deg = metadata_dict[radar_utils.LAT_SPACING_COLUMN]
longitude_spacing_deg = metadata_dict[radar_utils.LNG_SPACING_COLUMN]
# TODO(thunderhoser): The following "if" condition is a hack. The purpose
# is to change grid corners only for actual MYRORSS data, not GridRad data
# in MYRORSS format.
if latitude_spacing_deg < 0.011 and longitude_spacing_deg < 0.011:
metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN] = (
rounder.floor_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN],
metadata_dict[radar_utils.LAT_SPACING_COLUMN]))
metadata_dict[radar_utils.NW_GRID_POINT_LNG_COLUMN] = (
rounder.ceiling_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LNG_COLUMN],
metadata_dict[radar_utils.LNG_SPACING_COLUMN]))
sentinel_values = []
for this_column in SENTINEL_VALUE_COLUMNS_ORIG:
sentinel_values.append(getattr(netcdf_dataset, this_column))
metadata_dict.update(
{radar_utils.SENTINEL_VALUE_COLUMN: numpy.array(sentinel_values)})
netcdf_dataset.close()
return metadata_dict
def _run(input_file_name, predictor_colour_map_name,
min_colour_prctile_for_predictors, max_colour_prctile_for_predictors,
saliency_colour_map_name, max_colour_prctile_for_saliency,
saliency_contour_line_width, num_saliency_contours, output_dir_name):
"""Plots saliency maps.
This is effectively the main method.
:param input_file_name: See documentation at top of file.
:param predictor_colour_map_name: Same.
:param min_colour_prctile_for_predictors: Same.
:param max_colour_prctile_for_predictors: Same.
:param saliency_colour_map_name: Same.
:param max_colour_prctile_for_saliency: Same.
:param saliency_contour_line_width: Same.
:param num_saliency_contours: Same.
:param output_dir_name: Same.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
error_checking.assert_is_geq(min_colour_prctile_for_predictors, 0.)
error_checking.assert_is_leq(max_colour_prctile_for_predictors, 100.)
error_checking.assert_is_greater(max_colour_prctile_for_predictors,
min_colour_prctile_for_predictors)
error_checking.assert_is_geq(max_colour_prctile_for_saliency, 0.)
error_checking.assert_is_leq(max_colour_prctile_for_saliency, 100.)
error_checking.assert_is_geq(num_saliency_contours, 2)
num_saliency_contours = 1 + int(
number_rounding.floor_to_nearest(num_saliency_contours, 2))
half_num_saliency_contours = (num_saliency_contours - 1) / 2
predictor_colour_map_object = pyplot.cm.get_cmap(predictor_colour_map_name)
saliency_colour_map_object = pyplot.cm.get_cmap(saliency_colour_map_name)
print 'Reading data from: "{0:s}"...'.format(input_file_name)
predictor_matrix, saliency_matrix, saliency_metadata_dict = (
saliency_maps.read_file(input_file_name))
model_metafile_name = traditional_cnn.find_metafile(
model_file_name=saliency_metadata_dict[
saliency_maps.MODEL_FILE_NAME_KEY])
print 'Reading metadata from: "{0:s}"...'.format(model_metafile_name)
model_metadata_dict = traditional_cnn.read_model_metadata(
model_metafile_name)
narr_predictor_names = model_metadata_dict[
traditional_cnn.NARR_PREDICTOR_NAMES_KEY]
num_predictors = len(narr_predictor_names)
num_examples = predictor_matrix.shape[0]
for i in range(num_examples):
this_min_cval_by_predictor = numpy.full(num_predictors, numpy.nan)
this_max_cval_by_predictor = this_min_cval_by_predictor + 0.
for k in range(num_predictors):
this_min_cval_by_predictor[k] = numpy.percentile(
predictor_matrix[i, ..., k], min_colour_prctile_for_predictors)
this_max_cval_by_predictor[k] = numpy.percentile(
predictor_matrix[i, ..., k], max_colour_prctile_for_predictors)
_, these_axes_objects = example_plotting.plot_many_predictors_sans_barbs(
predictor_matrix=predictor_matrix[i, ...],
predictor_names=narr_predictor_names,
cmap_object_by_predictor=[predictor_colour_map_object] *
num_predictors,
min_colour_value_by_predictor=this_min_cval_by_predictor,
max_colour_value_by_predictor=this_max_cval_by_predictor)
this_max_abs_contour_level = numpy.percentile(
numpy.absolute(saliency_matrix[i, ...]),
max_colour_prctile_for_saliency)
this_contour_interval = (this_max_abs_contour_level /
half_num_saliency_contours)
saliency_plotting.plot_many_2d_grids(
saliency_matrix_3d=saliency_matrix[i, ...],
axes_objects_2d_list=these_axes_objects,
colour_map_object=saliency_colour_map_object,
max_absolute_contour_level=this_max_abs_contour_level,
contour_interval=this_contour_interval,
line_width=saliency_contour_line_width)
this_figure_file_name = '{0:s}/example{1:06d}_saliency.jpg'.format(
output_dir_name, i)
print 'Saving figure to: "{0:s}"...'.format(this_figure_file_name)
pyplot.savefig(this_figure_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
def _plot_histogram(num_fronts_by_bin, front_type_string, title_string,
annotation_string, output_file_name):
"""Plots histogram of either cold-front or warm-front lengths.
B = number of bins
:param num_fronts_by_bin: length-B numpy array with number of fronts in each
bin.
:param front_type_string: Front type (must be accepted by
`front_utils.check_front_type`).
:param title_string: Figure title (will be placed above the figure).
:param annotation_string: Text annotation (will be placed in top left of
figure).
:param output_file_name: Path to output file (figure will be saved here).
"""
bin_frequencies = (num_fronts_by_bin.astype(float) /
numpy.sum(num_fronts_by_bin))
num_bins = len(num_fronts_by_bin)
bin_edges_metres = numpy.linspace(MIN_HISTOGRAM_LENGTH_METRES,
MAX_HISTOGRAM_LENGTH_METRES,
num=num_bins + 1)
bin_width_metres = bin_edges_metres[1] - bin_edges_metres[0]
bin_centers_metres = bin_edges_metres[:-1] + bin_width_metres / 2
_, axes_object = pyplot.subplots(1,
1,
figsize=(FIGURE_WIDTH_INCHES,
FIGURE_HEIGHT_INCHES))
if front_type_string == front_utils.WARM_FRONT_STRING_ID:
this_colour = front_plotting.DEFAULT_WARM_FRONT_COLOUR
else:
this_colour = front_plotting.DEFAULT_COLD_FRONT_COLOUR
axes_object.bar(bin_centers_metres * METRES_TO_KM,
bin_frequencies,
bin_width_metres * METRES_TO_KM,
color=this_colour,
edgecolor=LINE_COLOUR,
linewidth=LINE_WIDTH)
max_y_tick_value = rounder.floor_to_nearest(
1.05 * numpy.max(bin_frequencies), Y_TICK_SPACING)
num_y_ticks = 1 + int(numpy.round(max_y_tick_value / Y_TICK_SPACING))
y_tick_values = numpy.linspace(0., max_y_tick_value, num=num_y_ticks)
pyplot.yticks(y_tick_values)
axes_object.set_xlim(MIN_HISTOGRAM_LENGTH_METRES * METRES_TO_KM,
MAX_HISTOGRAM_LENGTH_METRES * METRES_TO_KM)
axes_object.set_ylim(0., 1.05 * numpy.max(bin_frequencies))
x_tick_values, _ = pyplot.xticks()
x_tick_labels = []
for i in range(len(x_tick_values)):
this_label = '{0:d}'.format(int(numpy.round(x_tick_values[i])))
if i == len(x_tick_values) - 1:
this_label += '+'
x_tick_labels.append(this_label)
pyplot.xticks(x_tick_values, x_tick_labels)
pyplot.xlabel('Front length (km)')
pyplot.ylabel('Frequency')
pyplot.title(title_string)
plotting_utils.annotate_axes(axes_object=axes_object,
annotation_string=annotation_string)
print 'Saving figure to: "{0:s}"...'.format(output_file_name)
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
pyplot.savefig(output_file_name, dpi=OUTPUT_RESOLUTION_DPI)
pyplot.close()
imagemagick_utils.trim_whitespace(input_file_name=output_file_name,
output_file_name=output_file_name)
def _run(storm_metafile_name, top_tracking_dir_name, latitude_buffer_deg,
longitude_buffer_deg, rap_directory_name, ruc_directory_name,
lead_time_seconds, lag_time_seconds, field_name_grib1,
output_dir_name):
"""Plots RAP/RUC field centered on each example (storm object).
This is effectively the main method.
:param storm_metafile_name: See documentation at top of file.
:param top_tracking_dir_name: Same.
:param latitude_buffer_deg: Same.
:param longitude_buffer_deg: Same.
:param rap_directory_name: Same.
:param ruc_directory_name: Same.
:param lead_time_seconds: Same.
:param lag_time_seconds: Same.
:param field_name_grib1: Same.
:param output_dir_name: Same.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name
)
error_checking.assert_is_geq(latitude_buffer_deg, 1.)
error_checking.assert_is_geq(longitude_buffer_deg, 1.)
error_checking.assert_is_greater(lead_time_seconds, 0)
error_checking.assert_is_greater(lag_time_seconds, 0)
print('Reading metadata from: "{0:s}"...'.format(storm_metafile_name))
full_storm_id_strings, storm_times_unix_sec = (
tracking_io.read_ids_and_times(storm_metafile_name)
)
init_times_unix_sec = (
storm_times_unix_sec + lead_time_seconds - lag_time_seconds
)
init_times_unix_sec = number_rounding.floor_to_nearest(
init_times_unix_sec, INIT_TIME_INTERVAL_SEC
)
init_times_unix_sec = init_times_unix_sec.astype(int)
num_examples = len(full_storm_id_strings)
rap_file_names = [None] * num_examples
ruc_file_names = [None] * num_examples
for i in range(num_examples):
if init_times_unix_sec[i] >= FIRST_RAP_TIME_UNIX_SEC:
rap_file_names[i] = nwp_model_io.find_rap_file_any_grid(
top_directory_name=rap_directory_name,
init_time_unix_sec=init_times_unix_sec[i],
lead_time_hours=0, raise_error_if_missing=True
)
continue
ruc_file_names[i] = nwp_model_io.find_ruc_file_any_grid(
top_directory_name=ruc_directory_name,
init_time_unix_sec=init_times_unix_sec[i],
lead_time_hours=0, raise_error_if_missing=True
)
for i in range(num_examples):
_plot_rapruc_one_example(
full_storm_id_string=full_storm_id_strings[i],
storm_time_unix_sec=storm_times_unix_sec[i],
top_tracking_dir_name=top_tracking_dir_name,
latitude_buffer_deg=latitude_buffer_deg,
longitude_buffer_deg=longitude_buffer_deg,
lead_time_seconds=lead_time_seconds,
field_name_grib1=field_name_grib1, output_dir_name=output_dir_name,
rap_file_name=rap_file_names[i], ruc_file_name=ruc_file_names[i]
)
projection_object = projections.init_azimuthal_equidistant_projection(central_latitude_deg=central_latitude_deg,
central_longitude_deg=central_longitude_deg)
# Project lat-long grid points to x-y.
(grid_point_x_matrix_metres, grid_point_y_matrix_metres) = projections.project_latlng_to_xy(
latitudes_deg=lats,
longitudes_deg=lons,
projection_object=projection_object)
x_min_metres = numpy.min(grid_point_x_matrix_metres)
x_max_metres = numpy.max(grid_point_x_matrix_metres)
y_min_metres = numpy.min(grid_point_y_matrix_metres)
y_max_metres = numpy.max(grid_point_y_matrix_metres)
# Round corners to nearest 10 km. These will become the corners of the actual
# x-y grid.
x_min_metres = number_rounding.floor_to_nearest(x_min_metres, X_SPACING_METRES)
x_max_metres = number_rounding.ceiling_to_nearest(x_max_metres, X_SPACING_METRES)
y_min_metres = number_rounding.floor_to_nearest(y_min_metres, Y_SPACING_METRES)
y_max_metres = number_rounding.ceiling_to_nearest(y_max_metres, Y_SPACING_METRES)
num_grid_rows = 1 + int(numpy.round((y_max_metres - y_min_metres) / Y_SPACING_METRES))
num_grid_columns = 1 + int(numpy.round((x_max_metres - x_min_metres) / X_SPACING_METRES))
(unique_grid_point_x_metres, unique_grid_point_y_metres) = grids.get_xy_grid_points(
x_min_metres=x_min_metres,
y_min_metres=y_min_metres,
x_spacing_metres=X_SPACING_METRES,
y_spacing_metres=Y_SPACING_METRES,
num_rows=num_grid_rows, num_columns=num_grid_columns)
(grid_point_x_matrix_metres, grid_point_y_matrix_metres) = grids.xy_vectors_to_matrices(
def _plot_tornado_and_radar(top_myrorss_dir_name, radar_field_name,
radar_height_m_asl, spc_date_string, tornado_table,
tornado_row, output_file_name):
"""Plots one unlinked tornado with radar field.
:param top_myrorss_dir_name: See documentation at top of file.
:param radar_field_name: Same.
:param radar_height_m_asl: Same.
:param spc_date_string: SPC date for linkage file (format "yyyymmdd").
:param tornado_table: pandas DataFrame created by
`linkage._read_input_tornado_reports`.
:param tornado_row: Will plot only tornado in [j]th row of table, where j =
`tornado_row`.
:param output_file_name: Path to output file. Figure will be saved here.
"""
tornado_time_unix_sec = tornado_table[
linkage.EVENT_TIME_COLUMN].values[tornado_row]
radar_time_unix_sec = number_rounding.round_to_nearest(
tornado_time_unix_sec, RADAR_TIME_INTERVAL_SEC)
radar_spc_date_string = time_conversion.time_to_spc_date_string(
radar_time_unix_sec)
radar_file_name = myrorss_and_mrms_io.find_raw_file(
top_directory_name=top_myrorss_dir_name,
spc_date_string=radar_spc_date_string,
unix_time_sec=radar_time_unix_sec,
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_field_name,
height_m_asl=radar_height_m_asl,
raise_error_if_missing=spc_date_string == radar_spc_date_string)
if not os.path.isfile(radar_file_name):
first_radar_time_unix_sec = number_rounding.ceiling_to_nearest(
time_conversion.get_start_of_spc_date(spc_date_string),
RADAR_TIME_INTERVAL_SEC)
last_radar_time_unix_sec = number_rounding.floor_to_nearest(
time_conversion.get_end_of_spc_date(spc_date_string),
RADAR_TIME_INTERVAL_SEC)
radar_time_unix_sec = max(
[radar_time_unix_sec, first_radar_time_unix_sec])
radar_time_unix_sec = min(
[radar_time_unix_sec, last_radar_time_unix_sec])
radar_file_name = myrorss_and_mrms_io.find_raw_file(
top_directory_name=top_myrorss_dir_name,
spc_date_string=spc_date_string,
unix_time_sec=radar_time_unix_sec,
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_field_name,
height_m_asl=radar_height_m_asl,
raise_error_if_missing=True)
radar_metadata_dict = myrorss_and_mrms_io.read_metadata_from_raw_file(
netcdf_file_name=radar_file_name,
data_source=radar_utils.MYRORSS_SOURCE_ID)
sparse_grid_table = (myrorss_and_mrms_io.read_data_from_sparse_grid_file(
netcdf_file_name=radar_file_name,
field_name_orig=radar_metadata_dict[
myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG],
data_source=radar_utils.MYRORSS_SOURCE_ID,
sentinel_values=radar_metadata_dict[radar_utils.SENTINEL_VALUE_COLUMN])
)
radar_matrix, grid_point_latitudes_deg, grid_point_longitudes_deg = (
radar_s2f.sparse_to_full_grid(sparse_grid_table=sparse_grid_table,
metadata_dict=radar_metadata_dict))
radar_matrix = numpy.flip(radar_matrix, axis=0)
grid_point_latitudes_deg = grid_point_latitudes_deg[::-1]
axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=numpy.min(grid_point_latitudes_deg),
max_latitude_deg=numpy.max(grid_point_latitudes_deg),
min_longitude_deg=numpy.min(grid_point_longitudes_deg),
max_longitude_deg=numpy.max(grid_point_longitudes_deg),
resolution_string='i')[1:])
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS)
radar_plotting.plot_latlng_grid(
field_matrix=radar_matrix,
field_name=radar_field_name,
axes_object=axes_object,
min_grid_point_latitude_deg=numpy.min(grid_point_latitudes_deg),
min_grid_point_longitude_deg=numpy.min(grid_point_longitudes_deg),
latitude_spacing_deg=numpy.diff(grid_point_latitudes_deg[:2])[0],
longitude_spacing_deg=numpy.diff(grid_point_longitudes_deg[:2])[0])
tornado_latitude_deg = tornado_table[
linkage.EVENT_LATITUDE_COLUMN].values[tornado_row]
tornado_longitude_deg = tornado_table[
linkage.EVENT_LONGITUDE_COLUMN].values[tornado_row]
axes_object.plot(tornado_longitude_deg,
tornado_latitude_deg,
linestyle='None',
marker=TORNADO_MARKER_TYPE,
markersize=TORNADO_MARKER_SIZE,
markeredgewidth=TORNADO_MARKER_EDGE_WIDTH,
markerfacecolor=plotting_utils.colour_from_numpy_to_tuple(
TORNADO_MARKER_COLOUR),
markeredgecolor=plotting_utils.colour_from_numpy_to_tuple(
TORNADO_MARKER_COLOUR))
tornado_time_string = time_conversion.unix_sec_to_string(
tornado_time_unix_sec, TIME_FORMAT)
title_string = (
'Unlinked tornado at {0:s}, {1:.2f} deg N, {2:.2f} deg E').format(
tornado_time_string, tornado_latitude_deg, tornado_longitude_deg)
pyplot.title(title_string, fontsize=TITLE_FONT_SIZE)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
imagemagick_utils.trim_whitespace(input_file_name=output_file_name,
output_file_name=output_file_name)
def find_raw_azimuthal_shear_file(
desired_time_unix_sec, spc_date_unix_sec, field_name, data_source,
top_directory_name,
max_time_offset_sec=DEFAULT_MAX_TIME_OFFSET_FOR_AZ_SHEAR_SEC,
raise_error_if_missing=False):
"""Finds raw azimuthal-shear file on local machine.
If you know the exact time step for azimuthal shear, use find_raw_file.
However, azimuthal shear is "special" and its times are often offset from
those of other radar fields. This method accounts for that and finds
az-shear files within some offset of the desired time.
:param desired_time_unix_sec: Desired time for azimuthal shear.
:param spc_date_unix_sec: SPC date.
:param field_name: Field name in GewitterGefahr format (should match either
`LOW_LEVEL_SHEAR_NAME` or `MID_LEVEL_SHEAR_NAME`).
:param data_source: Data source (either "myrorss" or "mrms").
:param top_directory_name: Name of top-level directory with raw MYRORSS
files.
:param raise_error_if_missing: Boolean flag. If True and no az-shear file
can be found within `max_time_offset_sec` of `desired_time_unix_sec`,
will raise error. If False and no az-shear file can be found within
`max_time_offset_sec` of `desired_time_unix_sec`, will return None.
:return: raw_file_name: Path to raw az-shear file. If file is missing and
raise_error_if_missing = False, this is the *expected* path.
:raises: ValueError: if raise_error_if_missing = True and file is missing.
"""
error_checking.assert_is_integer(desired_time_unix_sec)
error_checking.assert_is_integer(max_time_offset_sec)
error_checking.assert_is_greater(max_time_offset_sec, 0)
error_checking.assert_is_boolean(raise_error_if_missing)
first_allowed_minute_unix_sec = numpy.round(int(rounder.floor_to_nearest(
float(desired_time_unix_sec - max_time_offset_sec),
MINUTES_TO_SECONDS)))
last_allowed_minute_unix_sec = numpy.round(int(rounder.floor_to_nearest(
float(desired_time_unix_sec + max_time_offset_sec),
MINUTES_TO_SECONDS)))
allowed_minutes_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_allowed_minute_unix_sec,
end_time_unix_sec=last_allowed_minute_unix_sec,
time_interval_sec=MINUTES_TO_SECONDS, include_endpoint=True).astype(int)
spc_date_string = time_conversion.time_to_spc_date_string(spc_date_unix_sec)
relative_directory_name = get_relative_dir_for_raw_files(
field_name=field_name, data_source=data_source)
raw_file_names = []
for this_time_unix_sec in allowed_minutes_unix_sec:
this_pathless_file_pattern = _get_pathless_raw_file_pattern(
this_time_unix_sec)
this_file_pattern = '{0:s}/{1:s}/{2:s}/{3:s}'.format(
top_directory_name, spc_date_string, relative_directory_name,
this_pathless_file_pattern)
raw_file_names += glob.glob(this_file_pattern)
file_times_unix_sec = []
for this_raw_file_name in raw_file_names:
_, this_pathless_file_name = os.path.split(this_raw_file_name)
this_time_string, ending = os.path.splitext(this_pathless_file_name)
if (ending.rfind("gz") > -1):
this_time_string, ending = os.path.splitext(this_time_string)
file_times_unix_sec.append(time_conversion.string_to_unix_sec(
this_time_string, TIME_FORMAT_SECONDS))
if len(file_times_unix_sec):
file_times_unix_sec = numpy.array(file_times_unix_sec)
time_differences_sec = numpy.absolute(
file_times_unix_sec - desired_time_unix_sec)
nearest_index = numpy.argmin(time_differences_sec)
min_time_diff_sec = time_differences_sec[nearest_index]
else:
min_time_diff_sec = numpy.inf
if min_time_diff_sec > max_time_offset_sec:
if raise_error_if_missing:
desired_time_string = time_conversion.unix_sec_to_string(
desired_time_unix_sec, TIME_FORMAT_FOR_LOG_MESSAGES)
log_string = ('Could not find "{0:s}" file within {1:d} seconds of '
'{2:s}').format(field_name, max_time_offset_sec,
desired_time_string)
raise ValueError(log_string)
return None
return raw_file_names[nearest_index]
def create_equidistant_grid(min_latitude_deg,
max_latitude_deg,
min_longitude_deg,
max_longitude_deg,
x_spacing_metres,
y_spacing_metres,
azimuthal=True):
"""Creates equidistant grid.
M = number of rows
N = number of columns
:param min_latitude_deg: Minimum latitude (deg N) in grid.
:param max_latitude_deg: Max latitude (deg N) in grid.
:param min_longitude_deg: Minimum longitude (deg E) in grid.
:param max_longitude_deg: Max longitude (deg E) in grid.
:param x_spacing_metres: Spacing between grid points in adjacent columns.
:param y_spacing_metres: Spacing between grid points in adjacent rows.
:param azimuthal: Boolean flag. If True, will create azimuthal equidistant
grid. If False, will create Lambert conformal grid.
:return: grid_dict: Dictionary with the following keys.
grid_dict['grid_point_x_coords_metres']: length-N numpy array with unique
x-coordinates at grid points.
grid_dict['grid_point_y_coords_metres']: length-M numpy array with unique
y-coordinates at grid points.
grid_dict['projection_object']: Instance of `pyproj.Proj` (used to convert
between lat-long coordinates and the x-y coordinates of the grid).
"""
# Check input args.
error_checking.assert_is_valid_latitude(min_latitude_deg)
error_checking.assert_is_valid_latitude(max_latitude_deg)
error_checking.assert_is_greater(max_latitude_deg, min_latitude_deg)
error_checking.assert_is_greater(x_spacing_metres, 0.)
error_checking.assert_is_greater(y_spacing_metres, 0.)
error_checking.assert_is_boolean(azimuthal)
min_longitude_deg = lng_conversion.convert_lng_negative_in_west(
min_longitude_deg, allow_nan=False)
max_longitude_deg = lng_conversion.convert_lng_negative_in_west(
max_longitude_deg, allow_nan=False)
error_checking.assert_is_greater(max_longitude_deg, min_longitude_deg)
# Create lat-long grid.
num_grid_rows = 1 + int(
numpy.round((max_latitude_deg - min_latitude_deg) /
DUMMY_LATITUDE_SPACING_DEG))
num_grid_columns = 1 + int(
numpy.round((max_longitude_deg - min_longitude_deg) /
DUMMY_LONGITUDE_SPACING_DEG))
unique_latitudes_deg, unique_longitudes_deg = get_latlng_grid_points(
min_latitude_deg=min_latitude_deg,
min_longitude_deg=min_longitude_deg,
lat_spacing_deg=DUMMY_LATITUDE_SPACING_DEG,
lng_spacing_deg=DUMMY_LONGITUDE_SPACING_DEG,
num_rows=num_grid_rows,
num_columns=num_grid_columns)
latitude_matrix_deg, longitude_matrix_deg = latlng_vectors_to_matrices(
unique_latitudes_deg=unique_latitudes_deg,
unique_longitudes_deg=unique_longitudes_deg)
# Create projection.
central_latitude_deg = 0.5 * (min_latitude_deg + max_latitude_deg)
central_longitude_deg = 0.5 * (min_longitude_deg + max_longitude_deg)
if azimuthal:
projection_object = projections.init_azimuthal_equidistant_projection(
central_latitude_deg=central_latitude_deg,
central_longitude_deg=central_longitude_deg)
else:
projection_object = projections.init_lcc_projection(
standard_latitudes_deg=numpy.full(2, central_latitude_deg),
central_longitude_deg=central_longitude_deg)
# Convert lat-long grid to preliminary x-y grid.
prelim_x_matrix_metres, prelim_y_matrix_metres = (
projections.project_latlng_to_xy(latitudes_deg=latitude_matrix_deg,
longitudes_deg=longitude_matrix_deg,
projection_object=projection_object))
# Find corners of preliminary x-y grid.
x_min_metres = numpy.min(prelim_x_matrix_metres)
x_max_metres = numpy.max(prelim_x_matrix_metres)
y_min_metres = numpy.min(prelim_y_matrix_metres)
y_max_metres = numpy.max(prelim_y_matrix_metres)
# Find corners of final x-y grid.
x_min_metres = number_rounding.floor_to_nearest(x_min_metres,
x_spacing_metres)
x_max_metres = number_rounding.ceiling_to_nearest(x_max_metres,
x_spacing_metres)
y_min_metres = number_rounding.floor_to_nearest(y_min_metres,
y_spacing_metres)
y_max_metres = number_rounding.ceiling_to_nearest(y_max_metres,
y_spacing_metres)
# Create final x-y grid.
num_grid_rows = 1 + int(
numpy.round((y_max_metres - y_min_metres) / y_spacing_metres))
num_grid_columns = 1 + int(
numpy.round((x_max_metres - x_min_metres) / x_spacing_metres))
unique_x_coords_metres, unique_y_coords_metres = get_xy_grid_points(
x_min_metres=x_min_metres,
y_min_metres=y_min_metres,
x_spacing_metres=x_spacing_metres,
y_spacing_metres=y_spacing_metres,
num_rows=num_grid_rows,
num_columns=num_grid_columns)
return {
X_COORDS_KEY: unique_x_coords_metres,
Y_COORDS_KEY: unique_y_coords_metres,
PROJECTION_KEY: projection_object
}
def read_metadata_from_raw_file(netcdf_file_name, data_source=None,
raise_error_if_fails=True):
"""Reads metadata raw (either MYRORSS or MRMS) file..
This file should contain one radar field at one height and one time step.
:param netcdf_file_name: Path to input file.
:param data_source: Data source (either "myrorss" or "mrms").
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error. If False and file cannot be
opened, this method will return None.
:return: metadata_dict: Dictionary with the following keys.
metadata_dict['nw_grid_point_lat_deg']: Latitude (deg N) of northwesternmost
grid point.
metadata_dict['nw_grid_point_lng_deg']: Longitude (deg E) of
northwesternmost grid point.
metadata_dict['lat_spacing_deg']: Spacing (deg N) between adjacent rows.
metadata_dict['lng_spacing_deg']: Spacing (deg E) between adjacent columns.
metadata_dict['num_lat_in_grid']: Number of rows (unique grid-point
latitudes).
metadata_dict['num_lng_in_grid']: Number of columns (unique grid-point
longitudes).
metadata_dict['height_m_agl']: Height (metres above ground level).
metadata_dict['unix_time_sec']: Time in Unix format.
metadata_dict['field_name']: Name of radar field in new format.
metadata_dict['field_name_orig']: Name of radar field in original (MYRORSS
or MRMS) format.
metadata_dict['sentinel_values']: 1-D numpy array of sentinel values.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
field_name_orig = str(getattr(netcdf_dataset, FIELD_NAME_COLUMN_ORIG))
metadata_dict = {
NW_GRID_POINT_LAT_COLUMN: getattr(netcdf_dataset,
NW_GRID_POINT_LAT_COLUMN_ORIG),
NW_GRID_POINT_LNG_COLUMN: lng_conversion.convert_lng_positive_in_west(
getattr(netcdf_dataset, NW_GRID_POINT_LNG_COLUMN_ORIG),
allow_nan=False),
LAT_SPACING_COLUMN: getattr(netcdf_dataset, LAT_SPACING_COLUMN_ORIG),
LNG_SPACING_COLUMN: getattr(netcdf_dataset, LNG_SPACING_COLUMN_ORIG),
NUM_LAT_COLUMN: netcdf_dataset.dimensions[NUM_LAT_COLUMN_ORIG].size + 1,
NUM_LNG_COLUMN: netcdf_dataset.dimensions[NUM_LNG_COLUMN_ORIG].size + 1,
HEIGHT_COLUMN: getattr(netcdf_dataset, HEIGHT_COLUMN_ORIG),
UNIX_TIME_COLUMN: getattr(netcdf_dataset, UNIX_TIME_COLUMN_ORIG),
FIELD_NAME_COLUMN_ORIG: field_name_orig,
FIELD_NAME_COLUMN: _field_name_orig_to_new(field_name_orig,
data_source=data_source)}
metadata_dict[NW_GRID_POINT_LAT_COLUMN] = rounder.floor_to_nearest(
metadata_dict[NW_GRID_POINT_LAT_COLUMN],
metadata_dict[LAT_SPACING_COLUMN])
metadata_dict[NW_GRID_POINT_LNG_COLUMN] = rounder.ceiling_to_nearest(
metadata_dict[NW_GRID_POINT_LNG_COLUMN],
metadata_dict[LNG_SPACING_COLUMN])
sentinel_values = numpy.full(len(SENTINEL_VALUE_COLUMNS_ORIG), numpy.nan)
for i in range(len(SENTINEL_VALUE_COLUMNS_ORIG)):
sentinel_values[i] = getattr(netcdf_dataset,
SENTINEL_VALUE_COLUMNS_ORIG[i])
metadata_dict.update({SENTINEL_VALUE_COLUMN: sentinel_values})
netcdf_dataset.close()
return metadata_dict
def get_times_needed_for_interp(query_times_unix_sec, model_time_step_hours,
method_string):
"""Finds model times needed for interp to each query time.
Q = number of query times
M = number of model times needed
:param query_times_unix_sec: length-Q numpy array of query times.
:param model_time_step_hours: Model time step. If interpolating between
forecast times from the same run, this should be time between successive
forecasts. If interpolating between model runs, this should be time
between successive model runs.
:param method_string: Interpolation method.
:return: model_times_unix_sec: length-M numpy array of model times needed.
:return: query_to_model_times_table: pandas DataFrame with the following
columns. Each row corresponds to a range of query times.
query_to_model_times_table.min_query_time_unix_sec: Earliest query time in
range.
query_to_model_times_table.max_query_time_unix_sec: Latest query time in
range.
query_to_model_times_table.these_model_times_unix_sec: 1-D numpy array of
model times needed for this range.
query_to_model_times_table.model_time_needed_flags: length-M numpy array of
Boolean flags. If model_time_needed_flags[j] = True,
model_times_unix_sec[j] -- or the [j]th element in the first output --
is needed for this range of query times.
"""
error_checking.assert_is_integer_numpy_array(query_times_unix_sec)
error_checking.assert_is_numpy_array(query_times_unix_sec,
num_dimensions=1)
error_checking.assert_is_string(method_string)
model_time_step_hours = int(numpy.round(model_time_step_hours))
model_time_step_sec = model_time_step_hours * HOURS_TO_SECONDS
min_min_query_time_unix_sec = rounder.floor_to_nearest(
float(numpy.min(query_times_unix_sec)), model_time_step_sec)
max_max_query_time_unix_sec = rounder.ceiling_to_nearest(
float(numpy.max(query_times_unix_sec)), model_time_step_sec)
if max_max_query_time_unix_sec == min_min_query_time_unix_sec:
max_max_query_time_unix_sec += model_time_step_sec
num_ranges = int(
numpy.round(
(max_max_query_time_unix_sec - min_min_query_time_unix_sec) /
model_time_step_sec))
min_query_times_unix_sec = numpy.linspace(min_min_query_time_unix_sec,
max_max_query_time_unix_sec -
model_time_step_sec,
num=num_ranges,
dtype=int)
max_query_times_unix_sec = numpy.linspace(min_min_query_time_unix_sec +
model_time_step_sec,
max_max_query_time_unix_sec,
num=num_ranges,
dtype=int)
if method_string == PREVIOUS_INTERP_METHOD:
min_model_time_unix_sec = min_min_query_time_unix_sec + 0
max_model_time_unix_sec = (max_max_query_time_unix_sec -
model_time_step_sec)
elif method_string == NEXT_INTERP_METHOD:
min_model_time_unix_sec = (min_min_query_time_unix_sec +
model_time_step_sec)
max_model_time_unix_sec = max_max_query_time_unix_sec + 0
elif method_string in SUPERLINEAR_INTERP_METHODS:
min_model_time_unix_sec = (min_min_query_time_unix_sec -
model_time_step_sec)
max_model_time_unix_sec = (max_max_query_time_unix_sec +
model_time_step_sec)
else:
min_model_time_unix_sec = min_min_query_time_unix_sec + 0
max_model_time_unix_sec = max_max_query_time_unix_sec + 0
num_model_times = 1 + int(
numpy.round((max_model_time_unix_sec - min_model_time_unix_sec) /
model_time_step_sec))
model_times_unix_sec = numpy.linspace(min_model_time_unix_sec,
max_model_time_unix_sec,
num=num_model_times,
dtype=int)
query_to_model_times_dict = {
MIN_QUERY_TIME_COLUMN: min_query_times_unix_sec,
MAX_QUERY_TIME_COLUMN: max_query_times_unix_sec
}
query_to_model_times_table = pandas.DataFrame.from_dict(
query_to_model_times_dict)
nested_array = query_to_model_times_table[[
MIN_QUERY_TIME_COLUMN, MIN_QUERY_TIME_COLUMN
]].values.tolist()
query_to_model_times_table = query_to_model_times_table.assign(
**{
MODEL_TIMES_COLUMN: nested_array,
MODEL_TIMES_NEEDED_COLUMN: nested_array
})
for i in range(num_ranges):
if method_string == PREVIOUS_INTERP_METHOD:
these_model_times_unix_sec = min_query_times_unix_sec[[i]]
elif method_string == NEXT_INTERP_METHOD:
these_model_times_unix_sec = max_query_times_unix_sec[[i]]
elif method_string in SUPERLINEAR_INTERP_METHODS:
these_model_times_unix_sec = numpy.array([
min_query_times_unix_sec[i] - model_time_step_sec,
min_query_times_unix_sec[i], max_query_times_unix_sec[i],
max_query_times_unix_sec[i] + model_time_step_sec
],
dtype=int)
else:
these_model_times_unix_sec = numpy.array(
[min_query_times_unix_sec[i], max_query_times_unix_sec[i]],
dtype=int)
query_to_model_times_table[MODEL_TIMES_COLUMN].values[i] = (
these_model_times_unix_sec)
query_to_model_times_table[MODEL_TIMES_NEEDED_COLUMN].values[i] = (
numpy.array([
t in these_model_times_unix_sec for t in model_times_unix_sec
],
dtype=bool))
return model_times_unix_sec, query_to_model_times_table
def find_raw_file_inexact_time(desired_time_unix_sec,
spc_date_string,
field_name,
data_source,
top_directory_name,
height_m_asl=None,
max_time_offset_sec=None,
raise_error_if_missing=False):
"""Finds raw file at inexact time.
If you know the exact valid time, use `find_raw_file`.
:param desired_time_unix_sec: Desired valid time.
:param spc_date_string: SPC date (format "yyyymmdd").
:param field_name: Field name in GewitterGefahr format.
:param data_source: Data source (string).
:param top_directory_name: Name of top-level directory with raw files.
:param height_m_asl: Radar height (metres above sea level).
:param max_time_offset_sec: Maximum offset between actual and desired valid
time.
For example, if `desired_time_unix_sec` is 162933 UTC 5 Jan 2018 and
`max_time_offset_sec` = 60, this method will look for az-shear at valid
times from 162833...163033 UTC 5 Jan 2018.
If None, this defaults to `DEFAULT_MAX_TIME_OFFSET_FOR_AZ_SHEAR_SEC` for
azimuthal-shear fields and `DEFAULT_MAX_TIME_OFFSET_FOR_NON_SHEAR_SEC` for
all other fields.
:param raise_error_if_missing: Boolean flag. If no file is found and
raise_error_if_missing = True, this method will error out. If no file
is found and raise_error_if_missing = False, will return None.
:return: raw_file_name: Path to raw file.
:raises: ValueError: if no file is found and raise_error_if_missing = True.
"""
# Error-checking.
error_checking.assert_is_integer(desired_time_unix_sec)
_ = time_conversion.spc_date_string_to_unix_sec(spc_date_string)
error_checking.assert_is_boolean(raise_error_if_missing)
radar_utils.check_field_name(field_name)
if max_time_offset_sec is None:
if field_name in AZIMUTHAL_SHEAR_FIELD_NAMES:
max_time_offset_sec = DEFAULT_MAX_TIME_OFFSET_FOR_AZ_SHEAR_SEC
else:
max_time_offset_sec = DEFAULT_MAX_TIME_OFFSET_FOR_NON_SHEAR_SEC
error_checking.assert_is_integer(max_time_offset_sec)
error_checking.assert_is_greater(max_time_offset_sec, 0)
first_allowed_minute_unix_sec = numpy.round(
int(
rounder.floor_to_nearest(
float(desired_time_unix_sec - max_time_offset_sec),
MINUTES_TO_SECONDS)))
last_allowed_minute_unix_sec = numpy.round(
int(
rounder.floor_to_nearest(
float(desired_time_unix_sec + max_time_offset_sec),
MINUTES_TO_SECONDS)))
allowed_minutes_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_allowed_minute_unix_sec,
end_time_unix_sec=last_allowed_minute_unix_sec,
time_interval_sec=MINUTES_TO_SECONDS,
include_endpoint=True).astype(int)
relative_directory_name = get_relative_dir_for_raw_files(
field_name=field_name,
data_source=data_source,
height_m_asl=height_m_asl)
raw_file_names = []
for this_time_unix_sec in allowed_minutes_unix_sec:
this_pathless_file_pattern = _get_pathless_raw_file_pattern(
this_time_unix_sec)
this_file_pattern = '{0:s}/{1:s}/{2:s}/{3:s}/{4:s}'.format(
top_directory_name, spc_date_string[:4], spc_date_string,
relative_directory_name, this_pathless_file_pattern)
raw_file_names += glob.glob(this_file_pattern)
file_times_unix_sec = []
for this_raw_file_name in raw_file_names:
file_times_unix_sec.append(raw_file_name_to_time(this_raw_file_name))
if len(file_times_unix_sec):
file_times_unix_sec = numpy.array(file_times_unix_sec)
time_differences_sec = numpy.absolute(file_times_unix_sec -
desired_time_unix_sec)
nearest_index = numpy.argmin(time_differences_sec)
min_time_diff_sec = time_differences_sec[nearest_index]
else:
min_time_diff_sec = numpy.inf
if min_time_diff_sec > max_time_offset_sec:
if raise_error_if_missing:
desired_time_string = time_conversion.unix_sec_to_string(
desired_time_unix_sec, TIME_FORMAT_FOR_LOG_MESSAGES)
error_string = (
'Could not find "{0:s}" file within {1:d} seconds of {2:s}.'
).format(field_name, max_time_offset_sec, desired_time_string)
raise ValueError(error_string)
return None
return raw_file_names[nearest_index]
def get_times_needed_for_interp(query_times_unix_sec=None,
model_time_step_hours=None,
method_string=None):
"""Finds model times needed for interpolation to each range of query times.
Q = number of query times
M = number of model times needed
:param query_times_unix_sec: length-Q numpy array of query times (Unix
format).
:param model_time_step_hours: Model time step. If interpolating between
forecast times (from the same initialization), this should be the
model's time resolution (hours between successive forecasts). If
interpolating between model runs (forecasts for the same valid time but
from different initializations), this should be the model's refresh time
(hours between successive model runs).
:param method_string: Interpolation method. Valid options are "previous",
"next", "linear", "nearest", "zero", "slinear", "quadratic", and
"cubic". The last 6 methods are described in the documentation for
`scipy.interpolate.interp1d`.
:return: model_times_unix_sec: length-M numpy array of model times needed
(Unix format).
:return: query_to_model_times_table: pandas DataFrame with the following
columns. Each row corresponds to one range of query times.
query_to_model_times_table.min_query_time_unix_sec: Minimum query time for
this range.
query_to_model_times_table.max_query_time_unix_sec: Max query time for this
range.
query_to_model_times_table.model_times_unix_sec: 1-D numpy array of model
times needed for this range.
query_to_model_times_table.model_time_needed_flags: length-M numpy array of
Boolean flags. If model_time_needed_flags[i] = True at row j, this
means the [i]th model time is needed for interp to the [j]th range of
query times.
"""
error_checking.assert_is_integer_numpy_array(query_times_unix_sec)
error_checking.assert_is_numpy_array_without_nan(query_times_unix_sec)
error_checking.assert_is_numpy_array(query_times_unix_sec,
num_dimensions=1)
model_time_step_hours = int(numpy.round(model_time_step_hours))
error_checking.assert_is_string(method_string)
model_time_step_sec = model_time_step_hours * HOURS_TO_SECONDS
min_min_query_time_unix_sec = rounder.floor_to_nearest(
float(numpy.min(query_times_unix_sec)), model_time_step_sec)
max_max_query_time_unix_sec = rounder.ceiling_to_nearest(
float(numpy.max(query_times_unix_sec)), model_time_step_sec)
if max_max_query_time_unix_sec == min_min_query_time_unix_sec:
max_max_query_time_unix_sec += model_time_step_sec
num_ranges = int(
(max_max_query_time_unix_sec - min_min_query_time_unix_sec) /
model_time_step_sec)
min_query_times_unix_sec = numpy.linspace(min_min_query_time_unix_sec,
max_max_query_time_unix_sec -
model_time_step_sec,
num=num_ranges,
dtype=int)
max_query_times_unix_sec = numpy.linspace(min_min_query_time_unix_sec +
model_time_step_sec,
max_max_query_time_unix_sec,
num=num_ranges,
dtype=int)
if method_string == PREVIOUS_INTERP_METHOD:
min_model_time_unix_sec = copy.deepcopy(min_min_query_time_unix_sec)
max_model_time_unix_sec = (max_max_query_time_unix_sec -
model_time_step_sec)
elif method_string == NEXT_INTERP_METHOD:
min_model_time_unix_sec = (min_min_query_time_unix_sec +
model_time_step_sec)
max_model_time_unix_sec = copy.deepcopy(max_max_query_time_unix_sec)
elif method_string in SUPERLINEAR_INTERP_METHODS:
min_model_time_unix_sec = (min_min_query_time_unix_sec -
model_time_step_sec)
max_model_time_unix_sec = (max_max_query_time_unix_sec +
model_time_step_sec)
else:
min_model_time_unix_sec = copy.deepcopy(min_min_query_time_unix_sec)
max_model_time_unix_sec = copy.deepcopy(max_max_query_time_unix_sec)
num_model_times = int((max_model_time_unix_sec - min_model_time_unix_sec) /
model_time_step_sec) + 1
model_times_unix_sec = numpy.linspace(min_model_time_unix_sec,
max_model_time_unix_sec,
num=num_model_times,
dtype=int)
query_to_model_times_dict = {
MIN_QUERY_TIME_COLUMN: min_query_times_unix_sec,
MAX_QUERY_TIME_COLUMN: max_query_times_unix_sec
}
query_to_model_times_table = pandas.DataFrame.from_dict(
query_to_model_times_dict)
nested_array = query_to_model_times_table[[
MIN_QUERY_TIME_COLUMN, MIN_QUERY_TIME_COLUMN
]].values.tolist()
argument_dict = {
MODEL_TIMES_COLUMN: nested_array,
MODEL_TIMES_NEEDED_COLUMN: nested_array
}
query_to_model_times_table = query_to_model_times_table.assign(
**argument_dict)
for i in range(num_ranges):
if method_string == PREVIOUS_INTERP_METHOD:
these_model_times_unix_sec = numpy.array(
[min_query_times_unix_sec[i]], dtype=int)
elif method_string == NEXT_INTERP_METHOD:
these_model_times_unix_sec = numpy.array(
[max_query_times_unix_sec[i]], dtype=int)
elif method_string in SUPERLINEAR_INTERP_METHODS:
these_model_times_unix_sec = numpy.array([
min_query_times_unix_sec[i] - model_time_step_sec,
min_query_times_unix_sec[i], max_query_times_unix_sec[i],
max_query_times_unix_sec[i] + model_time_step_sec
],
dtype=int)
else:
these_model_times_unix_sec = numpy.array(
[min_query_times_unix_sec[i], max_query_times_unix_sec[i]],
dtype=int)
query_to_model_times_table[MODEL_TIMES_COLUMN].values[
i] = these_model_times_unix_sec
query_to_model_times_table[MODEL_TIMES_NEEDED_COLUMN].values[i] = [
t in these_model_times_unix_sec for t in model_times_unix_sec
]
return model_times_unix_sec, query_to_model_times_table
def _plot_one_composite(gradcam_file_name, monte_carlo_file_name,
composite_name_abbrev, composite_name_verbose,
colour_map_object, min_colour_value, max_colour_value,
num_contours, smoothing_radius_grid_cells,
output_dir_name):
"""Plots class-activation map for one composite.
:param gradcam_file_name: Path to input file (will be read by
`gradcam.read_file`).
:param monte_carlo_file_name: Path to Monte Carlo file (will be read by
`_read_monte_carlo_file`).
:param composite_name_abbrev: Abbrev composite name (will be used in file
names).
:param composite_name_verbose: Verbose composite name (will be used in
figure title).
:param colour_map_object: See documentation at top of file.
:param min_colour_value: Minimum value in colour bar (may be NaN).
:param max_colour_value: Max value in colour bar (may be NaN).
:param num_contours: See documentation at top of file.
:param smoothing_radius_grid_cells: Same.
:param output_dir_name: Name of output directory (figures will be saved
here).
:return: main_figure_file_name: Path to main image file created by this
method.
:return: min_colour_value: Same as input but cannot be None.
:return: max_colour_value: Same as input but cannot be None.
"""
(mean_radar_matrix, mean_class_activn_matrix, significance_matrix,
model_metadata_dict) = _read_one_composite(
gradcam_file_name=gradcam_file_name,
smoothing_radius_grid_cells=smoothing_radius_grid_cells,
monte_carlo_file_name=monte_carlo_file_name)
print(numpy.percentile(mean_class_activn_matrix, 0.))
print(numpy.percentile(mean_class_activn_matrix, 1.))
print(numpy.percentile(mean_class_activn_matrix, 99.))
print(numpy.percentile(mean_class_activn_matrix, 100.))
if numpy.isnan(min_colour_value) or numpy.isnan(max_colour_value):
min_colour_value_log10 = number_rounding.floor_to_nearest(
numpy.log10(numpy.percentile(mean_class_activn_matrix, 1.)), 0.1)
max_colour_value_log10 = number_rounding.ceiling_to_nearest(
numpy.log10(numpy.percentile(mean_class_activn_matrix, 99.)), 0.1)
min_colour_value_log10 = max([min_colour_value_log10, -2.])
max_colour_value_log10 = max([max_colour_value_log10, -1.])
min_colour_value_log10 = min([min_colour_value_log10, 1.])
max_colour_value_log10 = min([max_colour_value_log10, 2.])
min_colour_value = 10**min_colour_value_log10
max_colour_value = 10**max_colour_value_log10
else:
min_colour_value_log10 = numpy.log10(min_colour_value)
max_colour_value_log10 = numpy.log10(max_colour_value)
contour_interval_log10 = (
(max_colour_value_log10 - min_colour_value_log10) / (num_contours - 1))
mean_activn_matrix_log10 = numpy.log10(mean_class_activn_matrix)
training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
field_names = training_option_dict[trainval_io.RADAR_FIELDS_KEY]
num_fields = mean_radar_matrix.shape[-1]
num_heights = mean_radar_matrix.shape[-2]
handle_dict = plot_examples.plot_one_example(
list_of_predictor_matrices=[mean_radar_matrix],
model_metadata_dict=model_metadata_dict,
pmm_flag=True,
allow_whitespace=True,
plot_panel_names=True,
panel_name_font_size=PANEL_NAME_FONT_SIZE,
add_titles=False,
label_colour_bars=True,
colour_bar_length=COLOUR_BAR_LENGTH,
colour_bar_font_size=COLOUR_BAR_FONT_SIZE,
num_panel_rows=num_heights)
figure_objects = handle_dict[plot_examples.RADAR_FIGURES_KEY]
axes_object_matrices = handle_dict[plot_examples.RADAR_AXES_KEY]
for k in range(num_fields):
cam_plotting.plot_many_2d_grids(
class_activation_matrix_3d=numpy.flip(
mean_activn_matrix_log10[0, ...], axis=0),
axes_object_matrix=axes_object_matrices[k],
colour_map_object=colour_map_object,
min_contour_level=min_colour_value_log10,
max_contour_level=max_colour_value_log10,
contour_interval=contour_interval_log10)
significance_plotting.plot_many_2d_grids_without_coords(
significance_matrix=numpy.flip(significance_matrix[0, ...],
axis=0),
axes_object_matrix=axes_object_matrices[k])
panel_file_names = [None] * num_fields
for k in range(num_fields):
panel_file_names[k] = '{0:s}/{1:s}_{2:s}.jpg'.format(
output_dir_name, composite_name_abbrev,
field_names[k].replace('_', '-'))
print('Saving figure to: "{0:s}"...'.format(panel_file_names[k]))
figure_objects[k].savefig(panel_file_names[k],
dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0,
bbox_inches='tight')
pyplot.close(figure_objects[k])
main_figure_file_name = '{0:s}/{1:s}_gradcam.jpg'.format(
output_dir_name, composite_name_abbrev)
print('Concatenating panels to: "{0:s}"...'.format(main_figure_file_name))
imagemagick_utils.concatenate_images(
input_file_names=panel_file_names,
output_file_name=main_figure_file_name,
num_panel_rows=1,
num_panel_columns=num_fields,
border_width_pixels=50)
imagemagick_utils.resize_image(input_file_name=main_figure_file_name,
output_file_name=main_figure_file_name,
output_size_pixels=CONCAT_FIGURE_SIZE_PX)
imagemagick_utils.trim_whitespace(input_file_name=main_figure_file_name,
output_file_name=main_figure_file_name,
border_width_pixels=TITLE_FONT_SIZE + 25)
_overlay_text(image_file_name=main_figure_file_name,
x_offset_from_center_px=0,
y_offset_from_top_px=0,
text_string=composite_name_verbose)
imagemagick_utils.trim_whitespace(input_file_name=main_figure_file_name,
output_file_name=main_figure_file_name,
border_width_pixels=10)
return main_figure_file_name, min_colour_value, max_colour_value
