def _compute_shape_stats(spc_date_string, top_tracking_dir_name,
tracking_scale_metres2, output_dir_name):
"""Computes shape statistics for each storm object.
:param spc_date_string: SPC (Storm Prediction Center) date in format
"yyyymmdd". Shape statistics will be computed for all storm objects on
this date.
:param top_tracking_dir_name: Name of top-level directory with storm-
tracking data.
:param tracking_scale_metres2: Tracking scale (minimum storm area). Will be
used to find input data.
:param output_dir_name: Name of output directory. A single Pickle file,
with shape statistics for each storm object, will be written here.
"""
tracking_file_names, _ = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)
print SEPARATOR_STRING
shape_statistic_table = shape_stats.get_stats_for_storm_objects(
storm_object_table)
print SEPARATOR_STRING
shape_statistic_file_name = '{0:s}/shape_statistics_{1:s}.p'.format(
output_dir_name, spc_date_string)
print 'Writing shape statistics to: "{0:s}"...'.format(
shape_statistic_file_name)
shape_stats.write_stats_for_storm_objects(shape_statistic_table,
shape_statistic_file_name)
def _compute_radar_stats_from_gridrad(spc_date_string, top_tracking_dir_name,
tracking_scale_metres2,
top_gridrad_dir_name, output_dir_name):
"""Uses GridRad data to compute radar statistics for each storm object.
:param spc_date_string: SPC (Storm Prediction Center) date in format
"yyyymmdd". Radar stats will be computed for all storm objects on this
date.
:param top_tracking_dir_name: Name of top-level directory with storm-
tracking files. Storm objects will be read from here.
:param tracking_scale_metres2: Tracking scale (minimum storm area). Will be
used to find tracking files.
:param top_gridrad_dir_name: Name of top-level directory with GridRad files.
:param output_dir_name: Name of output directory. A single Pickle file,
with radar stats for each storm object, will be written here.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
tracking_file_names, _ = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)
print SEPARATOR_STRING
storm_object_statistic_table = (
radar_statistics.get_storm_based_radar_stats_gridrad(
storm_object_table=storm_object_table,
top_radar_dir_name=top_gridrad_dir_name))
print SEPARATOR_STRING
output_file_name = '{0:s}/radar_stats_for_storm_objects_{1:s}.p'.format(
output_dir_name, spc_date_string)
print 'Writing radar statistics to file: "{0:s}"...'.format(
output_file_name)
radar_statistics.write_stats_for_storm_objects(
storm_object_statistic_table, output_file_name)
def get_storm_object_table(num_spc_dates, climatology_type, working_date_index):
date_in_memory_indices = _get_dates_needed(working_date_index, num_spc_dates, climatology_type)
for i in range(num_spc_dates):
if i in date_in_memory_indices:
if storm_object_table_by_spc_date[i] is None:
these_tracking_file_names = (tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_strings[i],
data_source='segmotion',
top_processed_dir_name=TOP_PROCESSED_DIR_NAME,
tracking_scale_metres2=TRACKING_SCALE_METRES2))
storm_object_table_by_spc_date[i] = (tracking_io.read_many_processed_files(
these_tracking_file_names))
else:
print 'Clearing data for SPC date "{0:s}"...'.format(spc_date_strings[i])
storm_object_table_by_spc_date[i] = None
for j in date_in_memory_indices[1:]:
storm_object_table_by_spc_date[j], _ = (storm_object_table_by_spc_date[j].align(
storm_object_table_by_spc_date[date_in_memory_indices[0]], axis=1))
storm_object_tables_to_concat = [storm_object_table_by_spc_date[j] for j in date_in_memory_indices]
multiday_storm_object_table = pandas.concat(storm_object_tables_to_concat, axis=0, ignore_index=True)
multiday_storm_object_table = multiday_storm_object_table[multiday_storm_object_table['age_sec']>= 900]
return multiday_storm_object_table
for i in range(num_spc_dates):
if i in date_in_memory_indices:
if storm_object_table_by_spc_date[i] is None:
# Find tracking files for [i]th date.
these_tracking_file_names = (
tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_strings[i],
data_source='segmotion',
top_processed_dir_name=TOP_PROCESSED_DIR_NAME,
tracking_scale_metres2=TRACKING_SCALE_METRES2))
# Read tracking files for [i]th date.
storm_object_table_by_spc_date[i] = (
tracking_io.read_many_processed_files(
these_tracking_file_names))
else:
print 'Clearing data for SPC date "{0:s}"...'.format(
spc_date_strings[i])
storm_object_table_by_spc_date[i] = None
print SEPARATOR_STRING
for j in date_in_memory_indices[1:]:
storm_object_table_by_spc_date[j], _ = (
storm_object_table_by_spc_date[j].align(
storm_object_table_by_spc_date[date_in_memory_indices[0]],
axis=1))
storm_object_tables_to_concat = [
def _extract_storm_images(num_image_rows, num_image_columns, rotate_grids,
rotated_grid_spacing_metres, radar_field_names,
radar_heights_m_agl, spc_date_string,
top_radar_dir_name, top_tracking_dir_name,
tracking_scale_metres2, target_name,
top_target_dir_name, top_output_dir_name):
"""Extracts storm-centered radar images from GridRad data.
:param num_image_rows: See documentation at top of file.
:param num_image_columns: Same.
:param rotate_grids: Same.
:param rotated_grid_spacing_metres: Same.
:param radar_field_names: Same.
:param radar_heights_m_agl: Same.
:param spc_date_string: Same.
:param top_radar_dir_name: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param target_name: Same.
:param top_target_dir_name: Same.
:param top_output_dir_name: Same.
"""
if target_name in ['', 'None']:
target_name = None
if target_name is not None:
target_param_dict = target_val_utils.target_name_to_params(target_name)
target_file_name = target_val_utils.find_target_file(
top_directory_name=top_target_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_string)
print 'Reading data from: "{0:s}"...'.format(target_file_name)
target_dict = target_val_utils.read_target_values(
netcdf_file_name=target_file_name, target_name=target_name)
print '\n'
# Find storm objects on the given SPC date.
tracking_file_names = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
# Read storm objects on the given SPC date.
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)[storm_images.STORM_COLUMNS_NEEDED]
print SEPARATOR_STRING
if target_name is not None:
print(
'Removing storm objects without target values (variable = '
'"{0:s}")...').format(target_name)
these_indices = tracking_utils.find_storm_objects(
all_storm_ids=storm_object_table[
tracking_utils.STORM_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.TIME_COLUMN].values.astype(int),
storm_ids_to_keep=target_dict[target_val_utils.STORM_IDS_KEY],
times_to_keep_unix_sec=target_dict[
target_val_utils.VALID_TIMES_KEY],
allow_missing=False)
num_storm_objects_orig = len(storm_object_table.index)
storm_object_table = storm_object_table.iloc[these_indices]
num_storm_objects = len(storm_object_table.index)
print 'Removed {0:d} of {1:d} storm objects!\n'.format(
num_storm_objects_orig - num_storm_objects, num_storm_objects_orig)
# Extract storm-centered radar images.
storm_images.extract_storm_images_gridrad(
storm_object_table=storm_object_table,
top_radar_dir_name=top_radar_dir_name,
top_output_dir_name=top_output_dir_name,
num_storm_image_rows=num_image_rows,
num_storm_image_columns=num_image_columns,
rotate_grids=rotate_grids,
rotated_grid_spacing_metres=rotated_grid_spacing_metres,
radar_field_names=radar_field_names,
radar_heights_m_agl=radar_heights_m_agl)
def _shuffle_io_for_renaming(input_file_names_by_date,
output_file_names_by_date,
valid_times_by_date_unix_sec,
storm_object_table_by_date, working_date_index):
"""Shuffles data into and out of memory for renaming storms.
For more on renaming storms, see doc for `rename_storms`.
N = number of dates
:param input_file_names_by_date: length-N list created by
`_find_io_files_for_renaming`.
:param output_file_names_by_date: Same.
:param valid_times_by_date_unix_sec: Same.
:param storm_object_table_by_date: length-N list, where the [i]th element is
a pandas DataFrame with tracking data for the [i]th date. At any given
time, all but 3 items should be None. Each table has columns documented
in `storm_tracking_io.write_processed_file`, plus the following column.
storm_object_table_by_date.date_index: Array index. If date_index[i] = j,
the [i]th row (storm object) comes from the [j]th date.
:param working_date_index: Index of date currently being processed. Only
dates (working_date_index - 1)...(working_date_index + 1) need to be in
memory. If None, this method will write/clear all data currently in
memory, without reading new data.
:return: storm_object_table_by_date: Same as input, except that different
items are filled and different items are None.
"""
num_dates = len(input_file_names_by_date)
if working_date_index is None:
date_needed_indices = numpy.array([-1], dtype=int)
else:
date_needed_indices = _get_dates_needed_for_renaming_storms(
working_date_index=working_date_index,
num_dates_in_period=num_dates)
for j in range(num_dates):
if j in date_needed_indices and storm_object_table_by_date[j] is None:
storm_object_table_by_date[j] = (
tracking_io.read_many_processed_files(
input_file_names_by_date[j].tolist()))
print SEPARATOR_STRING
this_num_storm_objects = len(storm_object_table_by_date[j].index)
these_date_indices = numpy.full(this_num_storm_objects,
j,
dtype=int)
argument_dict = {DATE_INDEX_KEY: these_date_indices}
storm_object_table_by_date[j] = storm_object_table_by_date[
j].assign(**argument_dict)
if j not in date_needed_indices:
if storm_object_table_by_date[j] is not None:
these_output_file_names = output_file_names_by_date[j]
these_valid_times_unix_sec = valid_times_by_date_unix_sec[j]
for k in range(len(these_valid_times_unix_sec)):
print 'Writing new data to "{0:s}"...'.format(
these_output_file_names[k])
these_indices = numpy.where(
storm_object_table_by_date[j][
tracking_utils.TIME_COLUMN].values ==
these_valid_times_unix_sec[k])[0]
tracking_io.write_processed_file(
storm_object_table=storm_object_table_by_date[j].
iloc[these_indices],
pickle_file_name=these_output_file_names[k])
print SEPARATOR_STRING
storm_object_table_by_date[j] = None
if working_date_index is None:
return storm_object_table_by_date
for j in date_needed_indices[1:]:
storm_object_table_by_date[j], _ = storm_object_table_by_date[j].align(
storm_object_table_by_date[j - 1], axis=1)
return storm_object_table_by_date
for working_date_index in range(num_spc_dates):
date_in_memory_indices = utils._get_dates_needed(working_date_index=working_date_index, num_dates=num_spc_dates,
climatology_type=PASSAGE_CLIMATOLOGY_TYPE)
for i in range(num_spc_dates):
if i in date_in_memory_indices:
if storm_object_table_by_spc_date[i] is None:
# Find tracking files for [i]th date.
these_tracking_file_names = (tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_strings[i],
data_source='segmotion',
top_processed_dir_name=TOP_PROCESSED_DIR_NAME,
tracking_scale_metres2=TRACKING_SCALE_METRES2))
# Read tracking files for [i]th date.
storm_object_table_by_spc_date[i] = (tracking_io.read_many_processed_files(
these_tracking_file_names))
else:
print 'Clearing data for SPC date "{0:s}"...'.format(spc_date_strings[i])
storm_object_table_by_spc_date[i] = None
#this is the part that initializes the grid array
for j in date_in_memory_indices[1:]:
storm_object_table_by_spc_date[j], _ = (storm_object_table_by_spc_date[j].align(
storm_object_table_by_spc_date[date_in_memory_indices[0]], axis=1))
storm_object_tables_to_concat = [storm_object_table_by_spc_date[j] for j in date_in_memory_indices]
multiday_storm_object_table = pandas.concat(storm_object_tables_to_concat, axis=0, ignore_index=True)
centroids_x_metres, centroids_y_metres = projections.project_latlng_to_xy(
latitudes_deg=multiday_storm_object_table[tracking_utils.CENTROID_LAT_COLUMN].values,
longitudes_deg=multiday_storm_object_table[tracking_utils.CENTROID_LNG_COLUMN].values,
projection_object=projection_object)
def _interp_soundings(spc_date_string, lead_times_seconds,
lag_time_for_convective_contamination_sec,
top_ruc_directory_name, top_rap_directory_name,
top_tracking_dir_name, tracking_scale_metres2,
top_output_dir_name):
"""Interpolates NWP sounding to each storm object at each lead time.
:param spc_date_string: See documentation at top of file.
:param lead_times_seconds: Same.
:param lag_time_for_convective_contamination_sec: Same.
:param top_ruc_directory_name: Same.
:param top_rap_directory_name: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param top_output_dir_name: Same.
:raises: ValueError: if model-initialization times needed are on opposite
sides of 0000 UTC 1 May 2012 (the cutoff between RUC and RAP models).
"""
lead_times_seconds = numpy.array(lead_times_seconds, dtype=int)
tracking_file_names, _ = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)
print SEPARATOR_STRING
first_storm_time_unix_sec = numpy.min(
storm_object_table[tracking_utils.TIME_COLUMN].values)
last_storm_time_unix_sec = numpy.max(
storm_object_table[tracking_utils.TIME_COLUMN].values)
first_init_time_unix_sec = number_rounding.floor_to_nearest(
(first_storm_time_unix_sec + numpy.min(lead_times_seconds) -
lag_time_for_convective_contamination_sec), HOURS_TO_SECONDS)
last_init_time_unix_sec = number_rounding.floor_to_nearest(
(last_storm_time_unix_sec + numpy.max(lead_times_seconds) -
lag_time_for_convective_contamination_sec), HOURS_TO_SECONDS)
extreme_init_times_unix_sec = numpy.array(
[first_init_time_unix_sec, last_init_time_unix_sec], dtype=int)
if numpy.all(extreme_init_times_unix_sec < FIRST_RAP_TIME_UNIX_SEC):
top_grib_directory_name = top_ruc_directory_name
model_name = nwp_model_utils.RUC_MODEL_NAME
elif numpy.all(extreme_init_times_unix_sec >= FIRST_RAP_TIME_UNIX_SEC):
top_grib_directory_name = top_rap_directory_name
model_name = nwp_model_utils.RAP_MODEL_NAME
else:
first_storm_time_string = time_conversion.unix_sec_to_string(
first_storm_time_unix_sec, STORM_TIME_FORMAT)
last_storm_time_string = time_conversion.unix_sec_to_string(
last_storm_time_unix_sec, STORM_TIME_FORMAT)
first_init_time_string = time_conversion.unix_sec_to_string(
first_init_time_unix_sec, MODEL_INIT_TIME_FORMAT)
last_init_time_string = time_conversion.unix_sec_to_string(
last_init_time_unix_sec, MODEL_INIT_TIME_FORMAT)
error_string = (
'First and last storm times are {0:s} and {1:s}. Thus, first and '
'last model-initialization times needed are {2:s} and {3:s}, which '
'are on opposite sides of {4:s} (the cutoff between RUC and RAP '
'models). The code is not generalized enough to interp data from '
'two different models. Sorry, eh?').format(
first_storm_time_string, last_storm_time_string,
first_init_time_string, last_init_time_string,
FIRST_RAP_TIME_STRING)
raise ValueError(error_string)
sounding_dict_by_lead_time = soundings.interp_soundings_to_storm_objects(
storm_object_table=storm_object_table,
top_grib_directory_name=top_grib_directory_name,
model_name=model_name,
use_all_grids=True,
height_levels_m_agl=soundings.DEFAULT_HEIGHT_LEVELS_M_AGL,
lead_times_seconds=lead_times_seconds,
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec,
wgrib_exe_name=WGRIB_EXE_NAME,
wgrib2_exe_name=WGRIB2_EXE_NAME,
raise_error_if_missing=False)
print SEPARATOR_STRING
num_lead_times = len(lead_times_seconds)
for k in range(num_lead_times):
this_sounding_file_name = soundings.find_sounding_file(
top_directory_name=top_output_dir_name,
spc_date_string=spc_date_string,
lead_time_seconds=lead_times_seconds[k],
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec,
raise_error_if_missing=False)
print 'Writing soundings to: "{0:s}"...'.format(
this_sounding_file_name)
soundings.write_soundings(
netcdf_file_name=this_sounding_file_name,
sounding_dict_height_coords=sounding_dict_by_lead_time[k],
lead_time_seconds=lead_times_seconds[k],
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec)
def _extract_storm_images(num_image_rows, num_image_columns, rotate_grids,
rotated_grid_spacing_metres, radar_field_names,
refl_heights_m_agl, spc_date_string,
tarred_myrorss_dir_name, untarred_myrorss_dir_name,
top_tracking_dir_name, tracking_scale_metres2,
target_name, top_target_dir_name,
top_output_dir_name):
"""Extracts storm-centered img for each field/height pair and storm object.
:param num_image_rows: See documentation at top of file.
:param num_image_columns: Same.
:param rotate_grids: Same.
:param rotated_grid_spacing_metres: Same.
:param radar_field_names: Same.
:param refl_heights_m_agl: Same.
:param spc_date_string: Same.
:param tarred_myrorss_dir_name: Same.
:param untarred_myrorss_dir_name: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param target_name: Same.
:param top_target_dir_name: Same.
:param top_output_dir_name: Same.
"""
if target_name in ['', 'None']:
target_name = None
if target_name is not None:
target_param_dict = target_val_utils.target_name_to_params(target_name)
target_file_name = target_val_utils.find_target_file(
top_directory_name=top_target_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_string)
print 'Reading data from: "{0:s}"...'.format(target_file_name)
target_dict = target_val_utils.read_target_values(
netcdf_file_name=target_file_name, target_name=target_name)
print '\n'
refl_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_utils.REFL_NAME)
# Untar files with azimuthal shear.
az_shear_field_names = list(
set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES))
if len(az_shear_field_names):
az_shear_tar_file_name = (
'{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar'.format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string))
myrorss_io.unzip_1day_tar_file(
tar_file_name=az_shear_tar_file_name,
field_names=az_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name)
print SEPARATOR_STRING
# Untar files with other radar fields.
non_shear_field_names = list(
set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES))
if len(non_shear_field_names):
non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=non_shear_tar_file_name,
field_names=non_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name,
refl_heights_m_asl=refl_heights_m_asl)
print SEPARATOR_STRING
# Read storm tracks for the given SPC date.
tracking_file_names = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)[storm_images.STORM_COLUMNS_NEEDED]
print SEPARATOR_STRING
if target_name is not None:
print(
'Removing storm objects without target values (variable = '
'"{0:s}")...').format(target_name)
these_indices = tracking_utils.find_storm_objects(
all_storm_ids=storm_object_table[
tracking_utils.STORM_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.TIME_COLUMN].values.astype(int),
storm_ids_to_keep=target_dict[target_val_utils.STORM_IDS_KEY],
times_to_keep_unix_sec=target_dict[
target_val_utils.VALID_TIMES_KEY],
allow_missing=False)
num_storm_objects_orig = len(storm_object_table.index)
storm_object_table = storm_object_table.iloc[these_indices]
num_storm_objects = len(storm_object_table.index)
print 'Removed {0:d} of {1:d} storm objects!\n'.format(
num_storm_objects_orig - num_storm_objects, num_storm_objects_orig)
# Extract storm-centered radar images.
storm_images.extract_storm_images_myrorss_or_mrms(
storm_object_table=storm_object_table,
radar_source=radar_utils.MYRORSS_SOURCE_ID,
top_radar_dir_name=untarred_myrorss_dir_name,
top_output_dir_name=top_output_dir_name,
num_storm_image_rows=num_image_rows,
num_storm_image_columns=num_image_columns,
rotate_grids=rotate_grids,
rotated_grid_spacing_metres=rotated_grid_spacing_metres,
radar_field_names=radar_field_names,
reflectivity_heights_m_agl=refl_heights_m_agl)
print SEPARATOR_STRING
# Remove untarred MYRORSS files.
myrorss_io.remove_unzipped_data_1day(
spc_date_string=spc_date_string,
top_directory_name=untarred_myrorss_dir_name,
field_names=radar_field_names,
refl_heights_m_asl=refl_heights_m_asl)
print SEPARATOR_STRING