def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string,
top_myrorss_dir_name, radar_field_name, output_file_name):
"""Evaluates a set of storm tracks.
This is effectively the main method.
:param top_tracking_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param top_myrorss_dir_name: Same.
:param radar_field_name: Same.
:param output_file_name: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
list_of_storm_object_tables = []
for this_spc_date_string in spc_date_strings:
these_file_names = tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=echo_top_tracking.
DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
if len(these_file_names) == 0:
continue
this_storm_object_table = tracking_io.read_many_files(
these_file_names)[STORM_OBJECT_COLUMNS]
list_of_storm_object_tables.append(this_storm_object_table)
if this_spc_date_string != spc_date_strings[-1]:
print(MINOR_SEPARATOR_STRING)
if len(list_of_storm_object_tables) == 1:
continue
list_of_storm_object_tables[-1] = list_of_storm_object_tables[
-1].align(list_of_storm_object_tables[0], axis=1)[0]
print(SEPARATOR_STRING)
storm_object_table = pandas.concat(list_of_storm_object_tables,
axis=0,
ignore_index=True)
evaluation_dict = tracking_eval.evaluate_tracks(
storm_object_table=storm_object_table,
top_myrorss_dir_name=top_myrorss_dir_name,
radar_field_name=radar_field_name)
print('Writing results to: "{0:s}"...'.format(output_file_name))
tracking_eval.write_file(evaluation_dict=evaluation_dict,
pickle_file_name=output_file_name)
def _compute_radar_stats_from_gridrad(spc_date_string, top_tracking_dir_name,
tracking_scale_metres2,
top_gridrad_dir_name, output_dir_name):
"""Uses GridRad data to compute radar statistics for each storm object.
:param spc_date_string: SPC (Storm Prediction Center) date in format
"yyyymmdd". Radar stats will be computed for all storm objects on this
date.
:param top_tracking_dir_name: Name of top-level directory with storm-
tracking files. Storm objects will be read from here.
:param tracking_scale_metres2: Tracking scale (minimum storm area). Will be
used to find tracking files.
:param top_gridrad_dir_name: Name of top-level directory with GridRad files.
:param output_dir_name: Name of output directory. A single Pickle file,
with radar stats for each storm object, will be written here.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
tracking_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_string,
source_name=tracking_utils.SEGMOTION_NAME,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
storm_object_table = tracking_io.read_many_files(tracking_file_names)
print(SEPARATOR_STRING)
storm_object_statistic_table = (
radar_statistics.get_storm_based_radar_stats_gridrad(
storm_object_table=storm_object_table,
top_radar_dir_name=top_gridrad_dir_name))
print(SEPARATOR_STRING)
output_file_name = '{0:s}/radar_stats_for_storm_objects_{1:s}.p'.format(
output_dir_name, spc_date_string)
print('Writing radar statistics to file: "{0:s}"...'.format(
output_file_name))
radar_statistics.write_stats_for_storm_objects(
storm_object_statistic_table, output_file_name)
def _compute_shape_stats(spc_date_string, top_tracking_dir_name,
tracking_scale_metres2, output_dir_name):
"""Computes shape statistics for each storm object.
:param spc_date_string: SPC (Storm Prediction Center) date in format
"yyyymmdd". Shape statistics will be computed for all storm objects on
this date.
:param top_tracking_dir_name: Name of top-level directory with storm-
tracking data.
:param tracking_scale_metres2: Tracking scale (minimum storm area). Will be
used to find input data.
:param output_dir_name: Name of output directory. A single Pickle file,
with shape statistics for each storm object, will be written here.
"""
tracking_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_string,
source_name=tracking_utils.SEGMOTION_NAME,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
storm_object_table = tracking_io.read_many_files(tracking_file_names)
print(SEPARATOR_STRING)
shape_statistic_table = shape_stats.get_stats_for_storm_objects(
storm_object_table)
print(SEPARATOR_STRING)
shape_statistic_file_name = '{0:s}/shape_statistics_{1:s}.p'.format(
output_dir_name, spc_date_string)
print('Writing shape statistics to: "{0:s}"...'.format(
shape_statistic_file_name))
shape_stats.write_stats_for_storm_objects(shape_statistic_table,
shape_statistic_file_name)
def _interp_soundings(spc_date_string, lead_times_seconds,
lag_time_for_convective_contamination_sec,
top_ruc_directory_name, top_rap_directory_name,
top_tracking_dir_name, tracking_scale_metres2,
top_output_dir_name):
"""Interpolates NWP sounding to each storm object at each lead time.
:param spc_date_string: See documentation at top of file.
:param lead_times_seconds: Same.
:param lag_time_for_convective_contamination_sec: Same.
:param top_ruc_directory_name: Same.
:param top_rap_directory_name: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param top_output_dir_name: Same.
:raises: ValueError: if model-initialization times needed are on opposite
sides of 0000 UTC 1 May 2012 (the cutoff between RUC and RAP models).
"""
lead_times_seconds = numpy.array(lead_times_seconds, dtype=int)
tracking_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_string,
source_name=tracking_utils.SEGMOTION_NAME,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
storm_object_table = tracking_io.read_many_files(tracking_file_names)
print(SEPARATOR_STRING)
first_storm_time_unix_sec = numpy.min(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
last_storm_time_unix_sec = numpy.max(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
first_init_time_unix_sec = number_rounding.floor_to_nearest(
(first_storm_time_unix_sec + numpy.min(lead_times_seconds) -
lag_time_for_convective_contamination_sec), HOURS_TO_SECONDS)
last_init_time_unix_sec = number_rounding.floor_to_nearest(
(last_storm_time_unix_sec + numpy.max(lead_times_seconds) -
lag_time_for_convective_contamination_sec), HOURS_TO_SECONDS)
extreme_init_times_unix_sec = numpy.array(
[first_init_time_unix_sec, last_init_time_unix_sec], dtype=int)
if numpy.all(extreme_init_times_unix_sec < FIRST_RAP_TIME_UNIX_SEC):
top_grib_directory_name = top_ruc_directory_name
model_name = nwp_model_utils.RUC_MODEL_NAME
elif numpy.all(extreme_init_times_unix_sec >= FIRST_RAP_TIME_UNIX_SEC):
top_grib_directory_name = top_rap_directory_name
model_name = nwp_model_utils.RAP_MODEL_NAME
else:
first_storm_time_string = time_conversion.unix_sec_to_string(
first_storm_time_unix_sec, STORM_TIME_FORMAT)
last_storm_time_string = time_conversion.unix_sec_to_string(
last_storm_time_unix_sec, STORM_TIME_FORMAT)
first_init_time_string = time_conversion.unix_sec_to_string(
first_init_time_unix_sec, MODEL_INIT_TIME_FORMAT)
last_init_time_string = time_conversion.unix_sec_to_string(
last_init_time_unix_sec, MODEL_INIT_TIME_FORMAT)
error_string = (
'First and last storm times are {0:s} and {1:s}. Thus, first and '
'last model-initialization times needed are {2:s} and {3:s}, which '
'are on opposite sides of {4:s} (the cutoff between RUC and RAP '
'models). The code is not generalized enough to interp data from '
'two different models. Sorry, eh?').format(
first_storm_time_string, last_storm_time_string,
first_init_time_string, last_init_time_string,
FIRST_RAP_TIME_STRING)
raise ValueError(error_string)
sounding_dict_by_lead_time = soundings.interp_soundings_to_storm_objects(
storm_object_table=storm_object_table,
top_grib_directory_name=top_grib_directory_name,
model_name=model_name,
use_all_grids=True,
height_levels_m_agl=soundings.DEFAULT_HEIGHT_LEVELS_M_AGL,
lead_times_seconds=lead_times_seconds,
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec,
wgrib_exe_name=WGRIB_EXE_NAME,
wgrib2_exe_name=WGRIB2_EXE_NAME,
raise_error_if_missing=False)
print(SEPARATOR_STRING)
num_lead_times = len(lead_times_seconds)
for k in range(num_lead_times):
this_sounding_file_name = soundings.find_sounding_file(
top_directory_name=top_output_dir_name,
spc_date_string=spc_date_string,
lead_time_seconds=lead_times_seconds[k],
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec,
raise_error_if_missing=False)
print(
'Writing soundings to: "{0:s}"...'.format(this_sounding_file_name))
soundings.write_soundings(
netcdf_file_name=this_sounding_file_name,
sounding_dict_height_coords=sounding_dict_by_lead_time[k],
lead_time_seconds=lead_times_seconds[k],
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec)
def _run(top_orig_tracking_dir_name, top_new_tracking_dir_name,
first_spc_date_string, last_spc_date_string, output_file_name):
"""Plots storms that were removed by remove_storms_outside_conus.py.
This is effectively the main method.
:param top_orig_tracking_dir_name: See documentation at top of file.
:param top_new_tracking_dir_name: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param output_file_name: Same.
"""
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
orig_tracking_file_names = []
for d in spc_date_strings:
orig_tracking_file_names += tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_orig_tracking_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=d,
raise_error_if_missing=False)[0]
valid_times_unix_sec = numpy.array(
[tracking_io.file_name_to_time(f) for f in orig_tracking_file_names],
dtype=int)
new_tracking_file_names = [
tracking_io.find_file(
top_tracking_dir_name=top_new_tracking_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
valid_time_unix_sec=t,
spc_date_string=time_conversion.time_to_spc_date_string(t),
raise_error_if_missing=True) for t in valid_times_unix_sec
]
orig_storm_object_table = tracking_io.read_many_files(
orig_tracking_file_names)
print(SEPARATOR_STRING)
new_storm_object_table = tracking_io.read_many_files(
new_tracking_file_names)
print(SEPARATOR_STRING)
orig_storm_id_strings = (
orig_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist())
orig_storm_times_unix_sec = (
orig_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
new_storm_id_strings = (
new_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist())
new_storm_times_unix_sec = (
new_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
num_orig_storm_objects = len(orig_storm_object_table.index)
orig_kept_flags = numpy.full(num_orig_storm_objects, 0, dtype=bool)
these_indices = tracking_utils.find_storm_objects(
all_id_strings=orig_storm_id_strings,
all_times_unix_sec=orig_storm_times_unix_sec,
id_strings_to_keep=new_storm_id_strings,
times_to_keep_unix_sec=new_storm_times_unix_sec,
allow_missing=False)
orig_kept_flags[these_indices] = True
orig_removed_indices = numpy.where(numpy.invert(orig_kept_flags))[0]
print('{0:d} of {1:d} storm objects were outside CONUS.'.format(
len(orig_removed_indices), num_orig_storm_objects))
removed_storm_object_table = orig_storm_object_table.iloc[
orig_removed_indices]
removed_latitudes_deg = removed_storm_object_table[
tracking_utils.CENTROID_LATITUDE_COLUMN].values
removed_longitudes_deg = removed_storm_object_table[
tracking_utils.CENTROID_LONGITUDE_COLUMN].values
figure_object, axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=numpy.min(removed_latitudes_deg) - 1.,
max_latitude_deg=numpy.max(removed_latitudes_deg) + 1.,
min_longitude_deg=numpy.min(removed_longitudes_deg) - 1.,
max_longitude_deg=numpy.max(removed_longitudes_deg) + 1.,
resolution_string='i'))
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS)
conus_latitudes_deg, conus_longitudes_deg = (
conus_boundary.read_from_netcdf())
conus_latitudes_deg, conus_longitudes_deg = conus_boundary.erode_boundary(
latitudes_deg=conus_latitudes_deg,
longitudes_deg=conus_longitudes_deg,
erosion_distance_metres=EROSION_DISTANCE_METRES)
axes_object.plot(conus_longitudes_deg,
conus_latitudes_deg,
color=LINE_COLOUR,
linestyle='solid',
linewidth=LINE_WIDTH)
axes_object.plot(removed_longitudes_deg,
removed_latitudes_deg,
linestyle='None',
marker=MARKER_TYPE,
markersize=MARKER_SIZE,
markeredgewidth=0,
markerfacecolor=MARKER_COLOUR,
markeredgecolor=MARKER_COLOUR)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
figure_object.savefig(output_file_name,
dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0,
bbox_inches='tight')
pyplot.close(figure_object)
def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string,
colour_map_name, min_plot_latitude_deg, max_plot_latitude_deg,
min_plot_longitude_deg, max_plot_longitude_deg, output_file_name):
"""Plots storm tracks for a continuous time period.
This is effectively the main method.
:param top_tracking_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param colour_map_name: Same.
:param min_plot_latitude_deg: Same.
:param max_plot_latitude_deg: Same.
:param min_plot_longitude_deg: Same.
:param max_plot_longitude_deg: Same.
:param output_file_name: Same.
"""
if colour_map_name in ['', 'None']:
colour_map_object = 'random'
else:
colour_map_object = pyplot.cm.get_cmap(colour_map_name)
if min_plot_latitude_deg <= SENTINEL_VALUE:
min_plot_latitude_deg = None
if max_plot_latitude_deg <= SENTINEL_VALUE:
max_plot_latitude_deg = None
if min_plot_longitude_deg <= SENTINEL_VALUE:
min_plot_longitude_deg = None
if max_plot_longitude_deg <= SENTINEL_VALUE:
max_plot_longitude_deg = None
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
list_of_storm_object_tables = []
for this_spc_date_string in spc_date_strings:
these_file_names = tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=echo_top_tracking.
DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
if len(these_file_names) == 0:
continue
this_storm_object_table = tracking_io.read_many_files(
these_file_names)[REQUIRED_COLUMNS]
list_of_storm_object_tables.append(this_storm_object_table)
if this_spc_date_string != spc_date_strings[-1]:
print(MINOR_SEPARATOR_STRING)
if len(list_of_storm_object_tables) == 1:
continue
list_of_storm_object_tables[-1] = list_of_storm_object_tables[
-1].align(list_of_storm_object_tables[0], axis=1)[0]
print(SEPARATOR_STRING)
storm_object_table = pandas.concat(list_of_storm_object_tables,
axis=0,
ignore_index=True)
if min_plot_latitude_deg is None:
min_plot_latitude_deg = numpy.min(
storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values
) - LATLNG_BUFFER_DEG
if max_plot_latitude_deg is None:
max_plot_latitude_deg = numpy.max(
storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values
) + LATLNG_BUFFER_DEG
if min_plot_longitude_deg is None:
min_plot_longitude_deg = numpy.min(
storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values
) - LATLNG_BUFFER_DEG
if max_plot_longitude_deg is None:
max_plot_longitude_deg = numpy.max(
storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values
) + LATLNG_BUFFER_DEG
_, axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg,
resolution_string='i'))
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS)
storm_plotting.plot_storm_tracks(storm_object_table=storm_object_table,
axes_object=axes_object,
basemap_object=basemap_object,
colour_map_object=colour_map_object)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string,
colour_map_name, min_plot_latitude_deg, max_plot_latitude_deg,
min_plot_longitude_deg, max_plot_longitude_deg, output_file_name):
"""Plots storm tracks for a continuous time period.
This is effectively the main method.
:param top_tracking_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param colour_map_name: Same.
:param min_plot_latitude_deg: Same.
:param max_plot_latitude_deg: Same.
:param min_plot_longitude_deg: Same.
:param max_plot_longitude_deg: Same.
:param output_file_name: Same.
"""
if colour_map_name in ['', 'None']:
colour_map_object = 'random'
else:
colour_map_object = pyplot.cm.get_cmap(colour_map_name)
if min_plot_latitude_deg <= SENTINEL_VALUE:
min_plot_latitude_deg = None
if max_plot_latitude_deg <= SENTINEL_VALUE:
max_plot_latitude_deg = None
if min_plot_longitude_deg <= SENTINEL_VALUE:
min_plot_longitude_deg = None
if max_plot_longitude_deg <= SENTINEL_VALUE:
max_plot_longitude_deg = None
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
list_of_storm_object_tables = []
for this_spc_date_string in spc_date_strings:
these_file_names = tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=echo_top_tracking.
DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
if len(these_file_names) == 0:
continue
this_storm_object_table = tracking_io.read_many_files(
these_file_names)[REQUIRED_COLUMNS]
list_of_storm_object_tables.append(this_storm_object_table)
if this_spc_date_string != spc_date_strings[-1]:
print(MINOR_SEPARATOR_STRING)
if len(list_of_storm_object_tables) == 1:
continue
list_of_storm_object_tables[-1] = list_of_storm_object_tables[
-1].align(list_of_storm_object_tables[0], axis=1)[0]
print(SEPARATOR_STRING)
storm_object_table = pandas.concat(list_of_storm_object_tables,
axis=0,
ignore_index=True)
# TODO(thunderhoser): HACK
first_time_unix_sec = time_conversion.string_to_unix_sec(
'2011-04-27-20', '%Y-%m-%d-%H')
storm_object_table = storm_object_table.loc[storm_object_table[
tracking_utils.VALID_TIME_COLUMN] >= first_time_unix_sec]
if min_plot_latitude_deg is None:
min_plot_latitude_deg = numpy.min(
storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values
) - LATLNG_BUFFER_DEG
if max_plot_latitude_deg is None:
max_plot_latitude_deg = numpy.max(
storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values
) + LATLNG_BUFFER_DEG
if min_plot_longitude_deg is None:
min_plot_longitude_deg = numpy.min(
storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values
) - LATLNG_BUFFER_DEG
if max_plot_longitude_deg is None:
max_plot_longitude_deg = numpy.max(
storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values
) + LATLNG_BUFFER_DEG
_, axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg,
resolution_string='i'))
# plotting_utils.plot_coastlines(
# basemap_object=basemap_object, axes_object=axes_object,
# line_colour=BORDER_COLOUR
# )
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS,
line_colour=numpy.full(3, 1.))
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS,
line_colour=numpy.full(3, 1.))
colour_bar_object = storm_plotting.plot_storm_tracks(
storm_object_table=storm_object_table,
axes_object=axes_object,
basemap_object=basemap_object,
colour_map_object=colour_map_object)
valid_times_unix_sec = (
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
# TODO(thunderhoser): HACK
tick_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=numpy.min(valid_times_unix_sec),
end_time_unix_sec=numpy.max(valid_times_unix_sec),
time_interval_sec=1800,
include_endpoint=True)
tick_time_strings = [
time_conversion.unix_sec_to_string(t, COLOUR_BAR_TIME_FORMAT)
for t in tick_times_unix_sec
]
colour_bar_object.set_ticks(tick_times_unix_sec)
colour_bar_object.set_ticklabels(tick_time_strings)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
pyplot.savefig(output_file_name,
dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0,
bbox_inches='tight')
pyplot.close()
def _extract_storm_images(
num_image_rows, num_image_columns, rotate_grids,
rotated_grid_spacing_metres, radar_field_names, radar_heights_m_agl,
spc_date_string, top_radar_dir_name, top_tracking_dir_name,
elevation_dir_name, tracking_scale_metres2, target_name,
top_target_dir_name, top_output_dir_name):
"""Extracts storm-centered radar images from GridRad data.
:param num_image_rows: See documentation at top of file.
:param num_image_columns: Same.
:param rotate_grids: Same.
:param rotated_grid_spacing_metres: Same.
:param radar_field_names: Same.
:param radar_heights_m_agl: Same.
:param spc_date_string: Same.
:param top_radar_dir_name: Same.
:param top_tracking_dir_name: Same.
:param elevation_dir_name: Same.
:param tracking_scale_metres2: Same.
:param target_name: Same.
:param top_target_dir_name: Same.
:param top_output_dir_name: Same.
"""
if target_name in ['', 'None']:
target_name = None
if target_name is not None:
target_param_dict = target_val_utils.target_name_to_params(target_name)
target_file_name = target_val_utils.find_target_file(
top_directory_name=top_target_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_string)
print('Reading data from: "{0:s}"...'.format(target_file_name))
target_dict = target_val_utils.read_target_values(
netcdf_file_name=target_file_name, target_names=[target_name]
)
print('\n')
# Find storm objects on the given SPC date.
tracking_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_string,
source_name=tracking_utils.SEGMOTION_NAME,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2
)[0]
# Read storm objects on the given SPC date.
storm_object_table = tracking_io.read_many_files(
tracking_file_names
)[storm_images.STORM_COLUMNS_NEEDED]
print(SEPARATOR_STRING)
if target_name is not None:
print((
'Removing storm objects without target values (variable = '
'"{0:s}")...'
).format(target_name))
these_indices = tracking_utils.find_storm_objects(
all_id_strings=storm_object_table[
tracking_utils.FULL_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.VALID_TIME_COLUMN].values.astype(int),
id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY],
times_to_keep_unix_sec=target_dict[
target_val_utils.VALID_TIMES_KEY],
allow_missing=False)
num_storm_objects_orig = len(storm_object_table.index)
storm_object_table = storm_object_table.iloc[these_indices]
num_storm_objects = len(storm_object_table.index)
print('Removed {0:d} of {1:d} storm objects!\n'.format(
num_storm_objects_orig - num_storm_objects, num_storm_objects_orig
))
# Extract storm-centered radar images.
storm_images.extract_storm_images_gridrad(
storm_object_table=storm_object_table,
top_radar_dir_name=top_radar_dir_name,
top_output_dir_name=top_output_dir_name,
elevation_dir_name=elevation_dir_name,
num_storm_image_rows=num_image_rows,
num_storm_image_columns=num_image_columns, rotate_grids=rotate_grids,
rotated_grid_spacing_metres=rotated_grid_spacing_metres,
radar_field_names=radar_field_names,
radar_heights_m_agl=radar_heights_m_agl)
def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string,
storm_colour, storm_opacity, include_secondary_ids,
min_plot_latitude_deg, max_plot_latitude_deg, min_plot_longitude_deg,
max_plot_longitude_deg, top_myrorss_dir_name, radar_field_name,
radar_height_m_asl, output_dir_name):
"""Plots storm outlines (along with IDs) at each time step.
This is effectively the main method.
:param top_tracking_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param storm_colour: Same.
:param storm_opacity: Same.
:param include_secondary_ids: Same.
:param min_plot_latitude_deg: Same.
:param max_plot_latitude_deg: Same.
:param min_plot_longitude_deg: Same.
:param max_plot_longitude_deg: Same.
:param top_myrorss_dir_name: Same.
:param radar_field_name: Same.
:param radar_height_m_asl: Same.
:param output_dir_name: Same.
"""
if top_myrorss_dir_name in ['', 'None']:
top_myrorss_dir_name = None
if radar_field_name != radar_utils.REFL_NAME:
radar_height_m_asl = None
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
tracking_file_names = []
for this_spc_date_string in spc_date_strings:
tracking_file_names += (tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
source_name=DUMMY_SOURCE_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0])
storm_object_table = tracking_io.read_many_files(tracking_file_names)
print(SEPARATOR_STRING)
latitude_limits_deg, longitude_limits_deg = _get_plotting_limits(
min_plot_latitude_deg=min_plot_latitude_deg,
max_plot_latitude_deg=max_plot_latitude_deg,
min_plot_longitude_deg=min_plot_longitude_deg,
max_plot_longitude_deg=max_plot_longitude_deg,
storm_object_table=storm_object_table)
min_plot_latitude_deg = latitude_limits_deg[0]
max_plot_latitude_deg = latitude_limits_deg[1]
min_plot_longitude_deg = longitude_limits_deg[0]
max_plot_longitude_deg = longitude_limits_deg[1]
valid_times_unix_sec = numpy.unique(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
num_times = len(valid_times_unix_sec)
for i in range(num_times):
these_current_rows = numpy.where(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values ==
valid_times_unix_sec[i])[0]
these_current_subrows = _filter_storm_objects_latlng(
storm_object_table=storm_object_table.iloc[these_current_rows],
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg)
if len(these_current_subrows) == 0:
continue
these_current_rows = these_current_rows[these_current_subrows]
this_storm_object_table = _find_relevant_storm_objects(
storm_object_table=storm_object_table,
current_rows=these_current_rows)
these_latlng_rows = _filter_storm_objects_latlng(
storm_object_table=this_storm_object_table,
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg)
if top_myrorss_dir_name is None:
this_radar_matrix = None
these_radar_latitudes_deg = None
these_radar_longitudes_deg = None
else:
this_myrorss_file_name = myrorss_and_mrms_io.find_raw_file(
top_directory_name=top_myrorss_dir_name,
unix_time_sec=valid_times_unix_sec[i],
spc_date_string=time_conversion.time_to_spc_date_string(
valid_times_unix_sec[i]),
field_name=radar_field_name,
data_source=radar_utils.MYRORSS_SOURCE_ID,
height_m_asl=radar_height_m_asl,
raise_error_if_missing=True)
print(
'Reading data from: "{0:s}"...'.format(this_myrorss_file_name))
this_metadata_dict = (
myrorss_and_mrms_io.read_metadata_from_raw_file(
netcdf_file_name=this_myrorss_file_name,
data_source=radar_utils.MYRORSS_SOURCE_ID))
this_sparse_grid_table = (
myrorss_and_mrms_io.read_data_from_sparse_grid_file(
netcdf_file_name=this_myrorss_file_name,
field_name_orig=this_metadata_dict[
myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG],
data_source=radar_utils.MYRORSS_SOURCE_ID,
sentinel_values=this_metadata_dict[
radar_utils.SENTINEL_VALUE_COLUMN]))
(this_radar_matrix, these_radar_latitudes_deg,
these_radar_longitudes_deg) = radar_s2f.sparse_to_full_grid(
sparse_grid_table=this_sparse_grid_table,
metadata_dict=this_metadata_dict)
this_radar_matrix = numpy.flipud(this_radar_matrix)
these_radar_latitudes_deg = these_radar_latitudes_deg[::-1]
_, this_axes_object, this_basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg,
resolution_string='i'))
_plot_storm_outlines_one_time(
storm_object_table=this_storm_object_table.iloc[these_latlng_rows],
valid_time_unix_sec=valid_times_unix_sec[i],
axes_object=this_axes_object,
basemap_object=this_basemap_object,
storm_colour=storm_colour,
storm_opacity=storm_opacity,
include_secondary_ids=include_secondary_ids,
output_dir_name=output_dir_name,
radar_matrix=this_radar_matrix,
radar_field_name=radar_field_name,
radar_latitudes_deg=these_radar_latitudes_deg,
radar_longitudes_deg=these_radar_longitudes_deg)
def _plot_one_example(full_id_string,
storm_time_unix_sec,
target_name,
forecast_probability,
tornado_dir_name,
top_tracking_dir_name,
top_myrorss_dir_name,
radar_field_name,
radar_height_m_asl,
latitude_buffer_deg,
longitude_buffer_deg,
top_output_dir_name,
aux_forecast_probabilities=None,
aux_activation_dict=None):
"""Plots one example with surrounding context at several times.
N = number of storm objects read from auxiliary activation file
:param full_id_string: Full storm ID.
:param storm_time_unix_sec: Storm time.
:param target_name: Name of target variable.
:param forecast_probability: Forecast tornado probability for this example.
:param tornado_dir_name: See documentation at top of file.
:param top_tracking_dir_name: Same.
:param top_myrorss_dir_name: Same.
:param radar_field_name: Same.
:param radar_height_m_asl: Same.
:param latitude_buffer_deg: Same.
:param longitude_buffer_deg: Same.
:param top_output_dir_name: Same.
:param aux_forecast_probabilities: length-N numpy array of forecast
probabilities. If this is None, will not plot forecast probs in maps.
:param aux_activation_dict: Dictionary returned by
`model_activation.read_file` from auxiliary file. If this is None, will
not plot forecast probs in maps.
"""
storm_time_string = time_conversion.unix_sec_to_string(
storm_time_unix_sec, TIME_FORMAT)
# Create output directory for this example.
output_dir_name = '{0:s}/{1:s}_{2:s}'.format(top_output_dir_name,
full_id_string,
storm_time_string)
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
# Find tracking files.
tracking_file_names = _find_tracking_files_one_example(
valid_time_unix_sec=storm_time_unix_sec,
top_tracking_dir_name=top_tracking_dir_name,
target_name=target_name)
tracking_times_unix_sec = numpy.array(
[tracking_io.file_name_to_time(f) for f in tracking_file_names],
dtype=int)
tracking_time_strings = [
time_conversion.unix_sec_to_string(t, TIME_FORMAT)
for t in tracking_times_unix_sec
]
# Read tracking files.
storm_object_table = tracking_io.read_many_files(tracking_file_names)
print('\n')
if aux_activation_dict is not None:
these_indices = tracking_utils.find_storm_objects(
all_id_strings=aux_activation_dict[model_activation.FULL_IDS_KEY],
all_times_unix_sec=aux_activation_dict[
model_activation.STORM_TIMES_KEY],
id_strings_to_keep=storm_object_table[
tracking_utils.FULL_ID_COLUMN].values.tolist(),
times_to_keep_unix_sec=storm_object_table[
tracking_utils.VALID_TIME_COLUMN].values,
allow_missing=True)
storm_object_probs = numpy.array([
aux_forecast_probabilities[k] if k >= 0 else numpy.nan
for k in these_indices
])
storm_object_table = storm_object_table.assign(
**{FORECAST_PROBABILITY_COLUMN: storm_object_probs})
primary_id_string = temporal_tracking.full_to_partial_ids([full_id_string
])[0][0]
this_storm_object_table = storm_object_table.loc[storm_object_table[
tracking_utils.PRIMARY_ID_COLUMN] == primary_id_string]
latitude_limits_deg, longitude_limits_deg = _get_plotting_limits(
storm_object_table=this_storm_object_table,
latitude_buffer_deg=latitude_buffer_deg,
longitude_buffer_deg=longitude_buffer_deg)
storm_min_latitudes_deg = numpy.array([
numpy.min(numpy.array(p.exterior.xy[1])) for p in storm_object_table[
tracking_utils.LATLNG_POLYGON_COLUMN].values
])
storm_max_latitudes_deg = numpy.array([
numpy.max(numpy.array(p.exterior.xy[1])) for p in storm_object_table[
tracking_utils.LATLNG_POLYGON_COLUMN].values
])
storm_min_longitudes_deg = numpy.array([
numpy.min(numpy.array(p.exterior.xy[0])) for p in storm_object_table[
tracking_utils.LATLNG_POLYGON_COLUMN].values
])
storm_max_longitudes_deg = numpy.array([
numpy.max(numpy.array(p.exterior.xy[0])) for p in storm_object_table[
tracking_utils.LATLNG_POLYGON_COLUMN].values
])
min_latitude_flags = numpy.logical_and(
storm_min_latitudes_deg >= latitude_limits_deg[0],
storm_min_latitudes_deg <= latitude_limits_deg[1])
max_latitude_flags = numpy.logical_and(
storm_max_latitudes_deg >= latitude_limits_deg[0],
storm_max_latitudes_deg <= latitude_limits_deg[1])
latitude_flags = numpy.logical_or(min_latitude_flags, max_latitude_flags)
min_longitude_flags = numpy.logical_and(
storm_min_longitudes_deg >= longitude_limits_deg[0],
storm_min_longitudes_deg <= longitude_limits_deg[1])
max_longitude_flags = numpy.logical_and(
storm_max_longitudes_deg >= longitude_limits_deg[0],
storm_max_longitudes_deg <= longitude_limits_deg[1])
longitude_flags = numpy.logical_or(min_longitude_flags,
max_longitude_flags)
good_indices = numpy.where(
numpy.logical_and(latitude_flags, longitude_flags))[0]
storm_object_table = storm_object_table.iloc[good_indices]
# Read tornado reports.
target_param_dict = target_val_utils.target_name_to_params(target_name)
min_lead_time_seconds = target_param_dict[
target_val_utils.MIN_LEAD_TIME_KEY]
max_lead_time_seconds = target_param_dict[
target_val_utils.MAX_LEAD_TIME_KEY]
tornado_table = linkage._read_input_tornado_reports(
input_directory_name=tornado_dir_name,
storm_times_unix_sec=numpy.array([storm_time_unix_sec], dtype=int),
max_time_before_storm_start_sec=-1 * min_lead_time_seconds,
max_time_after_storm_end_sec=max_lead_time_seconds,
genesis_only=True)
tornado_table = tornado_table.loc[
(tornado_table[linkage.EVENT_LATITUDE_COLUMN] >= latitude_limits_deg[0]
)
& (tornado_table[linkage.EVENT_LATITUDE_COLUMN] <=
latitude_limits_deg[1])]
tornado_table = tornado_table.loc[
(tornado_table[linkage.EVENT_LONGITUDE_COLUMN] >=
longitude_limits_deg[0])
& (tornado_table[linkage.EVENT_LONGITUDE_COLUMN] <=
longitude_limits_deg[1])]
for i in range(len(tracking_file_names)):
this_storm_object_table = storm_object_table.loc[storm_object_table[
tracking_utils.VALID_TIME_COLUMN] == tracking_times_unix_sec[i]]
_plot_one_example_one_time(
storm_object_table=this_storm_object_table,
full_id_string=full_id_string,
valid_time_unix_sec=tracking_times_unix_sec[i],
tornado_table=copy.deepcopy(tornado_table),
top_myrorss_dir_name=top_myrorss_dir_name,
radar_field_name=radar_field_name,
radar_height_m_asl=radar_height_m_asl,
latitude_limits_deg=latitude_limits_deg,
longitude_limits_deg=longitude_limits_deg)
if aux_activation_dict is None:
this_title_string = (
'Valid time = {0:s} ... forecast prob at {1:s} = {2:.3f}'
).format(tracking_time_strings[i], storm_time_string,
forecast_probability)
pyplot.title(this_title_string, fontsize=TITLE_FONT_SIZE)
this_file_name = '{0:s}/{1:s}.jpg'.format(output_dir_name,
tracking_time_strings[i])
print('Saving figure to file: "{0:s}"...\n'.format(this_file_name))
pyplot.savefig(this_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
imagemagick_utils.trim_whitespace(input_file_name=this_file_name,
output_file_name=this_file_name)
def _extract_storm_images(num_image_rows, num_image_columns, rotate_grids,
rotated_grid_spacing_metres, radar_field_names,
refl_heights_m_agl, spc_date_string,
tarred_myrorss_dir_name, untarred_myrorss_dir_name,
top_tracking_dir_name, elevation_dir_name,
tracking_scale_metres2, target_name,
top_target_dir_name, top_output_dir_name):
"""Extracts storm-centered img for each field/height pair and storm object.
:param num_image_rows: See documentation at top of file.
:param num_image_columns: Same.
:param rotate_grids: Same.
:param rotated_grid_spacing_metres: Same.
:param radar_field_names: Same.
:param refl_heights_m_agl: Same.
:param spc_date_string: Same.
:param tarred_myrorss_dir_name: Same.
:param untarred_myrorss_dir_name: Same.
:param top_tracking_dir_name: Same.
:param elevation_dir_name: Same.
:param tracking_scale_metres2: Same.
:param target_name: Same.
:param top_target_dir_name: Same.
:param top_output_dir_name: Same.
"""
if target_name in ['', 'None']:
target_name = None
if target_name is not None:
target_param_dict = target_val_utils.target_name_to_params(target_name)
target_file_name = target_val_utils.find_target_file(
top_directory_name=top_target_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_string)
print('Reading data from: "{0:s}"...'.format(target_file_name))
target_dict = target_val_utils.read_target_values(
netcdf_file_name=target_file_name, target_names=[target_name])
print('\n')
refl_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_utils.REFL_NAME)
# Untar files with azimuthal shear.
az_shear_field_names = list(
set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES))
if len(az_shear_field_names):
az_shear_tar_file_name = (
'{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar').format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=az_shear_tar_file_name,
field_names=az_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name)
print(SEPARATOR_STRING)
# Untar files with other radar fields.
non_shear_field_names = list(
set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES))
if len(non_shear_field_names):
non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=non_shear_tar_file_name,
field_names=non_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name,
refl_heights_m_asl=refl_heights_m_asl)
print(SEPARATOR_STRING)
# Read storm tracks for the given SPC date.
tracking_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_string,
source_name=tracking_utils.SEGMOTION_NAME,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
storm_object_table = tracking_io.read_many_files(tracking_file_names)[
storm_images.STORM_COLUMNS_NEEDED]
print(SEPARATOR_STRING)
if target_name is not None:
print(('Removing storm objects without target values (variable = '
'"{0:s}")...').format(target_name))
these_indices = tracking_utils.find_storm_objects(
all_id_strings=storm_object_table[
tracking_utils.FULL_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.VALID_TIME_COLUMN].values.astype(int),
id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY],
times_to_keep_unix_sec=target_dict[
target_val_utils.VALID_TIMES_KEY],
allow_missing=False)
num_storm_objects_orig = len(storm_object_table.index)
storm_object_table = storm_object_table.iloc[these_indices]
num_storm_objects = len(storm_object_table.index)
print('Removed {0:d} of {1:d} storm objects!\n'.format(
num_storm_objects_orig - num_storm_objects,
num_storm_objects_orig))
# Extract storm-centered radar images.
storm_images.extract_storm_images_myrorss_or_mrms(
storm_object_table=storm_object_table,
radar_source=radar_utils.MYRORSS_SOURCE_ID,
top_radar_dir_name=untarred_myrorss_dir_name,
top_output_dir_name=top_output_dir_name,
elevation_dir_name=elevation_dir_name,
num_storm_image_rows=num_image_rows,
num_storm_image_columns=num_image_columns,
rotate_grids=rotate_grids,
rotated_grid_spacing_metres=rotated_grid_spacing_metres,
radar_field_names=radar_field_names,
reflectivity_heights_m_agl=refl_heights_m_agl)
print(SEPARATOR_STRING)
# Remove untarred MYRORSS files.
myrorss_io.remove_unzipped_data_1day(
spc_date_string=spc_date_string,
top_directory_name=untarred_myrorss_dir_name,
field_names=radar_field_names,
refl_heights_m_asl=refl_heights_m_asl)
print(SEPARATOR_STRING)
def _extract_storm_images(
num_image_rows, num_image_columns, rotate_grids,
rotated_grid_spacing_metres, radar_field_names, refl_heights_m_agl,
spc_date_string, first_time_string, last_time_string,
tarred_myrorss_dir_name, untarred_myrorss_dir_name,
top_tracking_dir_name, elevation_dir_name, tracking_scale_metres2,
target_name, top_target_dir_name, top_output_dir_name):
"""Extracts storm-centered img for each field/height pair and storm object.
:param num_image_rows: See documentation at top of file.
:param num_image_columns: Same.
:param rotate_grids: Same.
:param rotated_grid_spacing_metres: Same.
:param radar_field_names: Same.
:param refl_heights_m_agl: Same.
:param spc_date_string: Same.
:param first_time_string: Same.
:param last_time_string: Same.
:param tarred_myrorss_dir_name: Same.
:param untarred_myrorss_dir_name: Same.
:param top_tracking_dir_name: Same.
:param elevation_dir_name: Same.
:param tracking_scale_metres2: Same.
:param target_name: Same.
:param top_target_dir_name: Same.
:param top_output_dir_name: Same.
:raises: ValueError: if `first_time_string` and `last_time_string` have
different SPC dates.
"""
if elevation_dir_name in ['', 'None']:
elevation_dir_name = None
if elevation_dir_name is None:
host_name = socket.gethostname()
if 'casper' in host_name:
elevation_dir_name = '/glade/work/ryanlage/elevation'
else:
elevation_dir_name = '/condo/swatwork/ralager/elevation'
if spc_date_string in ['', 'None']:
first_time_unix_sec = time_conversion.string_to_unix_sec(
first_time_string, TIME_FORMAT)
last_time_unix_sec = time_conversion.string_to_unix_sec(
last_time_string, TIME_FORMAT)
first_spc_date_string = time_conversion.time_to_spc_date_string(
first_time_unix_sec)
last_spc_date_string = time_conversion.time_to_spc_date_string(
last_time_unix_sec)
if first_spc_date_string != last_spc_date_string:
error_string = (
'First ({0:s}) and last ({1:s}) times have different SPC dates.'
' This script can handle only one SPC date.'
).format(first_time_string, last_time_string)
raise ValueError(error_string)
spc_date_string = first_spc_date_string
else:
first_time_unix_sec = 0
last_time_unix_sec = int(1e12)
if tarred_myrorss_dir_name in ['', 'None']:
tarred_myrorss_dir_name = None
if target_name in ['', 'None']:
target_name = None
if target_name is not None:
target_param_dict = target_val_utils.target_name_to_params(target_name)
target_file_name = target_val_utils.find_target_file(
top_directory_name=top_target_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_string)
print('Reading data from: "{0:s}"...'.format(target_file_name))
target_dict = target_val_utils.read_target_values(
netcdf_file_name=target_file_name, target_names=[target_name]
)
print('\n')
refl_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_utils.REFL_NAME)
# Untar files.
if tarred_myrorss_dir_name is not None:
az_shear_field_names = list(
set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES)
)
if len(az_shear_field_names) > 0:
az_shear_tar_file_name = (
'{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar'
).format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string
)
myrorss_io.unzip_1day_tar_file(
tar_file_name=az_shear_tar_file_name,
field_names=az_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name)
print(SEPARATOR_STRING)
non_shear_field_names = list(
set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES)
)
if len(non_shear_field_names) > 0:
non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string
)
myrorss_io.unzip_1day_tar_file(
tar_file_name=non_shear_tar_file_name,
field_names=non_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name,
refl_heights_m_asl=refl_heights_m_asl)
print(SEPARATOR_STRING)
# Read storm tracks for the given SPC date.
tracking_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_string,
source_name=tracking_utils.SEGMOTION_NAME,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2
)[0]
file_times_unix_sec = numpy.array(
[tracking_io.file_name_to_time(f) for f in tracking_file_names],
dtype=int
)
good_indices = numpy.where(numpy.logical_and(
file_times_unix_sec >= first_time_unix_sec,
file_times_unix_sec <= last_time_unix_sec
))[0]
tracking_file_names = [tracking_file_names[k] for k in good_indices]
storm_object_table = tracking_io.read_many_files(
tracking_file_names
)[storm_images.STORM_COLUMNS_NEEDED]
print(SEPARATOR_STRING)
if target_name is not None:
print((
'Removing storm objects without target values (variable = '
'"{0:s}")...'
).format(target_name))
these_indices = tracking_utils.find_storm_objects(
all_id_strings=storm_object_table[
tracking_utils.FULL_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.VALID_TIME_COLUMN].values.astype(int),
id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY],
times_to_keep_unix_sec=target_dict[
target_val_utils.VALID_TIMES_KEY],
allow_missing=False)
num_storm_objects_orig = len(storm_object_table.index)
storm_object_table = storm_object_table.iloc[these_indices]
num_storm_objects = len(storm_object_table.index)
print('Removed {0:d} of {1:d} storm objects!\n'.format(
num_storm_objects_orig - num_storm_objects, num_storm_objects_orig
))
# Extract storm-centered radar images.
storm_images.extract_storm_images_myrorss_or_mrms(
storm_object_table=storm_object_table,
radar_source=radar_utils.MYRORSS_SOURCE_ID,
top_radar_dir_name=untarred_myrorss_dir_name,
top_output_dir_name=top_output_dir_name,
elevation_dir_name=elevation_dir_name,
num_storm_image_rows=num_image_rows,
num_storm_image_columns=num_image_columns, rotate_grids=rotate_grids,
rotated_grid_spacing_metres=rotated_grid_spacing_metres,
radar_field_names=radar_field_names,
reflectivity_heights_m_agl=refl_heights_m_agl)
print(SEPARATOR_STRING)
# Remove untarred MYRORSS files.
if tarred_myrorss_dir_name is not None:
myrorss_io.remove_unzipped_data_1day(
spc_date_string=spc_date_string,
top_directory_name=untarred_myrorss_dir_name,
field_names=radar_field_names,
refl_heights_m_asl=refl_heights_m_asl)
def _run(input_prediction_file_name, top_tracking_dir_name,
tracking_scale_metres2, x_spacing_metres, y_spacing_metres,
effective_radius_metres, smoothing_method_name,
smoothing_cutoff_radius_metres, smoothing_efold_radius_metres,
top_output_dir_name):
"""Projects CNN forecasts onto the RAP grid.
This is effectively the same method.
:param input_prediction_file_name: See documentation at top of file.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param x_spacing_metres: Same.
:param y_spacing_metres: Same.
:param effective_radius_metres: Same.
:param smoothing_method_name: Same.
:param smoothing_cutoff_radius_metres: Same.
:param smoothing_efold_radius_metres: Same.
:param top_output_dir_name: Same.
"""
print('Reading data from: "{0:s}"...'.format(input_prediction_file_name))
ungridded_forecast_dict = prediction_io.read_ungridded_predictions(
input_prediction_file_name)
target_param_dict = target_val_utils.target_name_to_params(
ungridded_forecast_dict[prediction_io.TARGET_NAME_KEY])
min_buffer_dist_metres = target_param_dict[
target_val_utils.MIN_LINKAGE_DISTANCE_KEY]
# TODO(thunderhoser): This is HACKY.
if min_buffer_dist_metres == 0:
min_buffer_dist_metres = numpy.nan
max_buffer_dist_metres = target_param_dict[
target_val_utils.MAX_LINKAGE_DISTANCE_KEY]
min_lead_time_seconds = target_param_dict[
target_val_utils.MIN_LEAD_TIME_KEY]
max_lead_time_seconds = target_param_dict[
target_val_utils.MAX_LEAD_TIME_KEY]
forecast_column_name = gridded_forecasts._buffer_to_column_name(
min_buffer_dist_metres=min_buffer_dist_metres,
max_buffer_dist_metres=max_buffer_dist_metres,
column_type=gridded_forecasts.FORECAST_COLUMN_TYPE)
init_times_unix_sec = numpy.unique(
ungridded_forecast_dict[prediction_io.STORM_TIMES_KEY])
tracking_file_names = []
for this_time_unix_sec in init_times_unix_sec:
this_tracking_file_name = tracking_io.find_file(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
source_name=tracking_utils.SEGMOTION_NAME,
valid_time_unix_sec=this_time_unix_sec,
spc_date_string=time_conversion.time_to_spc_date_string(
this_time_unix_sec),
raise_error_if_missing=True)
tracking_file_names.append(this_tracking_file_name)
storm_object_table = tracking_io.read_many_files(tracking_file_names)
print(SEPARATOR_STRING)
tracking_utils.find_storm_objects(
all_id_strings=ungridded_forecast_dict[prediction_io.STORM_IDS_KEY],
all_times_unix_sec=ungridded_forecast_dict[
prediction_io.STORM_TIMES_KEY],
id_strings_to_keep=storm_object_table[
tracking_utils.FULL_ID_COLUMN].values.tolist(),
times_to_keep_unix_sec=storm_object_table[
tracking_utils.VALID_TIME_COLUMN].values,
allow_missing=False)
sort_indices = tracking_utils.find_storm_objects(
all_id_strings=storm_object_table[
tracking_utils.FULL_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.VALID_TIME_COLUMN].values,
id_strings_to_keep=ungridded_forecast_dict[
prediction_io.STORM_IDS_KEY],
times_to_keep_unix_sec=ungridded_forecast_dict[
prediction_io.STORM_TIMES_KEY],
allow_missing=False)
forecast_probabilities = ungridded_forecast_dict[
prediction_io.PROBABILITY_MATRIX_KEY][sort_indices, 1]
storm_object_table = storm_object_table.assign(
**{forecast_column_name: forecast_probabilities})
gridded_forecast_dict = gridded_forecasts.create_forecast_grids(
storm_object_table=storm_object_table,
min_lead_time_sec=min_lead_time_seconds,
max_lead_time_sec=max_lead_time_seconds,
lead_time_resolution_sec=gridded_forecasts.
DEFAULT_LEAD_TIME_RES_SECONDS,
grid_spacing_x_metres=x_spacing_metres,
grid_spacing_y_metres=y_spacing_metres,
interp_to_latlng_grid=False,
prob_radius_for_grid_metres=effective_radius_metres,
smoothing_method=smoothing_method_name,
smoothing_e_folding_radius_metres=smoothing_efold_radius_metres,
smoothing_cutoff_radius_metres=smoothing_cutoff_radius_metres)
print(SEPARATOR_STRING)
output_file_name = prediction_io.find_file(
top_prediction_dir_name=top_output_dir_name,
first_init_time_unix_sec=numpy.min(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values),
last_init_time_unix_sec=numpy.max(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values),
gridded=True,
raise_error_if_missing=False)
print(('Writing results (forecast grids for {0:d} initial times) to: '
'"{1:s}"...').format(
len(gridded_forecast_dict[prediction_io.INIT_TIMES_KEY]),
output_file_name))
prediction_io.write_gridded_predictions(
gridded_forecast_dict=gridded_forecast_dict,
pickle_file_name=output_file_name)
def _run(input_warning_file_name, top_tracking_dir_name, spc_date_string,
max_distance_metres, min_lifetime_fraction, output_warning_file_name):
"""Links each NWS tornado warning to nearest storm.
This is effectively the main method.
:param input_warning_file_name: See documentation at top of file.
:param top_tracking_dir_name: Same.
:param spc_date_string: Same.
:param max_distance_metres: Same.
:param min_lifetime_fraction: Same.
:param output_warning_file_name: Same.
"""
error_checking.assert_is_greater(max_distance_metres, 0.)
error_checking.assert_is_greater(min_lifetime_fraction, 0.)
error_checking.assert_is_leq(min_lifetime_fraction, 1.)
print('Reading warnings from: "{0:s}"...'.format(input_warning_file_name))
this_file_handle = open(input_warning_file_name, 'rb')
warning_table = pickle.load(this_file_handle)
this_file_handle.close()
date_start_time_unix_sec = (
time_conversion.get_start_of_spc_date(spc_date_string))
date_end_time_unix_sec = (
time_conversion.get_end_of_spc_date(spc_date_string))
warning_table = warning_table.loc[
(warning_table[WARNING_START_TIME_KEY] >= date_start_time_unix_sec)
& (warning_table[WARNING_START_TIME_KEY] <= date_end_time_unix_sec)]
num_warnings = len(warning_table.index)
print('Number of warnings beginning on SPC date "{0:s}" = {1:d}'.format(
spc_date_string, num_warnings))
warning_polygon_objects_xy = [None] * num_warnings
nested_array = warning_table[[
WARNING_START_TIME_KEY, WARNING_START_TIME_KEY
]].values.tolist()
warning_table = warning_table.assign(
**{
WARNING_XY_POLYGON_KEY: warning_polygon_objects_xy,
LINKED_SECONDARY_IDS_KEY: nested_array
})
for k in range(num_warnings):
warning_table[LINKED_SECONDARY_IDS_KEY].values[k] = []
this_object_latlng = warning_table[WARNING_LATLNG_POLYGON_KEY].values[
k]
warning_table[WARNING_XY_POLYGON_KEY].values[k], _ = (
polygons.project_latlng_to_xy(
polygon_object_latlng=this_object_latlng,
projection_object=PROJECTION_OBJECT))
tracking_file_names = []
for i in [-1, 0, 1]:
this_spc_date_string = time_conversion.time_to_spc_date_string(
date_start_time_unix_sec + i * NUM_SECONDS_PER_DAY)
# tracking_file_names += tracking_io.find_files_one_spc_date(
# top_tracking_dir_name=top_tracking_dir_name,
# tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
# source_name=tracking_utils.SEGMOTION_NAME,
# spc_date_string=this_spc_date_string,
# raise_error_if_missing=i == 0
# )[0]
tracking_file_names += tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
if len(tracking_file_names) == 0:
_write_linked_warnings(warning_table=warning_table,
output_file_name=output_warning_file_name)
return
print(SEPARATOR_STRING)
storm_object_table = tracking_io.read_many_files(tracking_file_names)
print(SEPARATOR_STRING)
if len(storm_object_table.index) == 0:
_write_linked_warnings(warning_table=warning_table,
output_file_name=output_warning_file_name)
return
storm_object_table = linkage._project_storms_latlng_to_xy(
storm_object_table=storm_object_table,
projection_object=PROJECTION_OBJECT)
for k in range(num_warnings):
this_start_time_string = time_conversion.unix_sec_to_string(
warning_table[WARNING_START_TIME_KEY].values[k],
LOG_MESSAGE_TIME_FORMAT)
this_end_time_string = time_conversion.unix_sec_to_string(
warning_table[WARNING_END_TIME_KEY].values[k],
LOG_MESSAGE_TIME_FORMAT)
print('Attempting to link warning from {0:s} to {1:s}...'.format(
this_start_time_string, this_end_time_string))
warning_table[LINKED_SECONDARY_IDS_KEY].values[k] = _link_one_warning(
warning_table=warning_table.iloc[[k]],
storm_object_table=copy.deepcopy(storm_object_table),
max_distance_metres=max_distance_metres,
min_lifetime_fraction=min_lifetime_fraction)
print('\n')
_write_linked_warnings(warning_table=warning_table,
output_file_name=output_warning_file_name)
def _shuffle_io_for_renaming(input_file_names_by_date,
output_file_names_by_date,
valid_times_by_date_unix_sec,
storm_object_table_by_date, working_date_index):
"""Shuffles data into and out of memory for renaming storms.
For more on renaming storms, see doc for `rename_storms`.
N = number of dates
:param input_file_names_by_date: length-N list created by
`_find_io_files_for_renaming`.
:param output_file_names_by_date: Same.
:param valid_times_by_date_unix_sec: Same.
:param storm_object_table_by_date: length-N list, where the [i]th element is
a pandas DataFrame with tracking data for the [i]th date. At any given
time, all but 3 items should be None. Each table has columns documented
in `storm_tracking_io.write_file`, plus the following column.
storm_object_table_by_date.date_index: Array index. If date_index[i] = j,
the [i]th row (storm object) comes from the [j]th date.
:param working_date_index: Index of date currently being processed. Only
dates (working_date_index - 1)...(working_date_index + 1) need to be in
memory. If None, this method will write/clear all data currently in
memory, without reading new data.
:return: storm_object_table_by_date: Same as input, except that different
items are filled and different items are None.
"""
num_dates = len(input_file_names_by_date)
if working_date_index is None:
date_needed_indices = numpy.array([-1], dtype=int)
else:
date_needed_indices = _get_dates_needed_for_renaming_storms(
working_date_index=working_date_index,
num_dates_in_period=num_dates)
for j in range(num_dates):
if j in date_needed_indices and storm_object_table_by_date[j] is None:
storm_object_table_by_date[j] = tracking_io.read_many_files(
input_file_names_by_date[j].tolist())
print(SEPARATOR_STRING)
this_num_storm_objects = len(storm_object_table_by_date[j].index)
these_date_indices = numpy.full(this_num_storm_objects,
j,
dtype=int)
storm_object_table_by_date[j] = (
storm_object_table_by_date[j].assign(
**{DATE_INDEX_KEY: these_date_indices}))
if j not in date_needed_indices:
if storm_object_table_by_date[j] is not None:
these_output_file_names = output_file_names_by_date[j]
these_valid_times_unix_sec = valid_times_by_date_unix_sec[j]
for k in range(len(these_valid_times_unix_sec)):
print('Writing new data to "{0:s}"...'.format(
these_output_file_names[k]))
these_indices = numpy.where(
storm_object_table_by_date[j][
tracking_utils.VALID_TIME_COLUMN].values ==
these_valid_times_unix_sec[k])[0]
tracking_io.write_file(
storm_object_table=storm_object_table_by_date[j].
iloc[these_indices],
pickle_file_name=these_output_file_names[k])
print(SEPARATOR_STRING)
storm_object_table_by_date[j] = None
if working_date_index is None:
return storm_object_table_by_date
for j in date_needed_indices[1:]:
storm_object_table_by_date[j], _ = storm_object_table_by_date[j].align(
storm_object_table_by_date[j - 1], axis=1)
return storm_object_table_by_date
