def _compute_shape_stats(spc_date_string, top_tracking_dir_name,
tracking_scale_metres2, output_dir_name):
"""Computes shape statistics for each storm object.
:param spc_date_string: SPC (Storm Prediction Center) date in format
"yyyymmdd". Shape statistics will be computed for all storm objects on
this date.
:param top_tracking_dir_name: Name of top-level directory with storm-
tracking data.
:param tracking_scale_metres2: Tracking scale (minimum storm area). Will be
used to find input data.
:param output_dir_name: Name of output directory. A single Pickle file,
with shape statistics for each storm object, will be written here.
"""
tracking_file_names, _ = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)
print SEPARATOR_STRING
shape_statistic_table = shape_stats.get_stats_for_storm_objects(
storm_object_table)
print SEPARATOR_STRING
shape_statistic_file_name = '{0:s}/shape_statistics_{1:s}.p'.format(
output_dir_name, spc_date_string)
print 'Writing shape statistics to: "{0:s}"...'.format(
shape_statistic_file_name)
shape_stats.write_stats_for_storm_objects(shape_statistic_table,
shape_statistic_file_name)
def _run(spc_date_string, top_wind_dir_name, top_tracking_dir_name,
tracking_scale_metres2, top_output_dir_name):
"""Runs `linkage.link_winds_to_storms`.
This is effectively the main method.
:param spc_date_string: See documentation at top of file.
:param top_wind_dir_name: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param top_output_dir_name: Same.
"""
tracking_file_names, _ = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
raise_error_if_missing=True)
storm_to_winds_table = linkage.link_storms_to_winds(
tracking_file_names=tracking_file_names,
top_wind_directory_name=top_wind_dir_name)
print SEPARATOR_STRING
output_file_name = linkage.find_linkage_file(
top_directory_name=top_output_dir_name,
event_type_string=linkage.WIND_EVENT_STRING,
spc_date_string=spc_date_string, raise_error_if_missing=False)
print 'Writing linkages to: "{0:s}"...'.format(output_file_name)
linkage.write_linkage_file(storm_to_events_table=storm_to_winds_table,
pickle_file_name=output_file_name)
def test_find_processed_files_one_spc_date(self):
"""Ensures correct output from find_processed_files_one_spc_date."""
_, this_glob_pattern = tracking_io.find_processed_files_one_spc_date(
top_processed_dir_name=TOP_SEGMOTION_DIR_NAME,
tracking_scale_metres2=TRACKING_SCALE_METRES2,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
spc_date_string=VALID_SPC_DATE_STRING,
raise_error_if_missing=False)
self.assertTrue(this_glob_pattern == GLOB_PATTERN_FOR_SPC_DATE)
def _write_csv_file_for_thea(
first_spc_date_string, last_spc_date_string, top_tracking_dir_name,
tracking_scale_metres2, output_file_name):
"""Converts storm tracks to a single CSV file for Thea.
:param first_spc_date_string: See documentation at top of file.
:param last_spc_date_string: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param output_file_name: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
tracking_file_names = []
for this_spc_date_string in spc_date_strings:
(these_tracking_file_names, _
) = tracking_io.find_processed_files_one_spc_date(
spc_date_string=this_spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)
tracking_file_names += these_tracking_file_names
num_files = len(tracking_file_names)
list_of_storm_object_tables = [None] * num_files
for i in range(num_files):
print 'Reading {0:d}th of {1:d} input files: "{2:s}"...'.format(
i + 1, num_files, tracking_file_names[i])
list_of_storm_object_tables[i] = tracking_io.read_processed_file(
tracking_file_names[i])
if i == 0:
continue
list_of_storm_object_tables[i], _ = (
list_of_storm_object_tables[i].align(
list_of_storm_object_tables[0], axis=1))
storm_object_table = pandas.concat(
list_of_storm_object_tables, axis=0, ignore_index=True)
print 'Writing all storm tracks to "{0:s}"...'.format(output_file_name)
best_tracks.write_simple_output_for_thea(
storm_object_table, output_file_name)
def _compute_radar_stats_from_gridrad(spc_date_string, top_tracking_dir_name,
tracking_scale_metres2,
top_gridrad_dir_name, output_dir_name):
"""Uses GridRad data to compute radar statistics for each storm object.
:param spc_date_string: SPC (Storm Prediction Center) date in format
"yyyymmdd". Radar stats will be computed for all storm objects on this
date.
:param top_tracking_dir_name: Name of top-level directory with storm-
tracking files. Storm objects will be read from here.
:param tracking_scale_metres2: Tracking scale (minimum storm area). Will be
used to find tracking files.
:param top_gridrad_dir_name: Name of top-level directory with GridRad files.
:param output_dir_name: Name of output directory. A single Pickle file,
with radar stats for each storm object, will be written here.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
tracking_file_names, _ = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)
print SEPARATOR_STRING
storm_object_statistic_table = (
radar_statistics.get_storm_based_radar_stats_gridrad(
storm_object_table=storm_object_table,
top_radar_dir_name=top_gridrad_dir_name))
print SEPARATOR_STRING
output_file_name = '{0:s}/radar_stats_for_storm_objects_{1:s}.p'.format(
output_dir_name, spc_date_string)
print 'Writing radar statistics to file: "{0:s}"...'.format(
output_file_name)
radar_statistics.write_stats_for_storm_objects(
storm_object_statistic_table, output_file_name)
def get_storm_object_table(num_spc_dates, climatology_type, working_date_index):
date_in_memory_indices = _get_dates_needed(working_date_index, num_spc_dates, climatology_type)
for i in range(num_spc_dates):
if i in date_in_memory_indices:
if storm_object_table_by_spc_date[i] is None:
these_tracking_file_names = (tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_strings[i],
data_source='segmotion',
top_processed_dir_name=TOP_PROCESSED_DIR_NAME,
tracking_scale_metres2=TRACKING_SCALE_METRES2))
storm_object_table_by_spc_date[i] = (tracking_io.read_many_processed_files(
these_tracking_file_names))
else:
print 'Clearing data for SPC date "{0:s}"...'.format(spc_date_strings[i])
storm_object_table_by_spc_date[i] = None
for j in date_in_memory_indices[1:]:
storm_object_table_by_spc_date[j], _ = (storm_object_table_by_spc_date[j].align(
storm_object_table_by_spc_date[date_in_memory_indices[0]], axis=1))
storm_object_tables_to_concat = [storm_object_table_by_spc_date[j] for j in date_in_memory_indices]
multiday_storm_object_table = pandas.concat(storm_object_tables_to_concat, axis=0, ignore_index=True)
multiday_storm_object_table = multiday_storm_object_table[multiday_storm_object_table['age_sec']>= 900]
return multiday_storm_object_table
def find_files_for_smart_io(start_time_unix_sec=None,
start_spc_date_string=None,
end_time_unix_sec=None,
end_spc_date_string=None,
data_source=None,
tracking_scale_metres2=None,
top_input_dir_name=None,
top_output_dir_name=None):
"""Finds input, output, and temporary working files for smart IO.
N = number of SPC dates in period
T_i = number of time steps in the [i]th SPC date
:param start_time_unix_sec: Beginning of time period.
:param start_spc_date_string: SPC date at beginning of time period (format
"yyyymmdd").
:param end_time_unix_sec: End of time period.
:param end_spc_date_string: SPC date at end of time period (format
"yyyymmdd").
:param data_source: Source for input data (examples: "segmotion",
"probSevere").
:param tracking_scale_metres2: Tracking scale.
:param top_input_dir_name: Name of top-level directory for input files.
:param top_output_dir_name: Name of top-level directory for output files.
:return: file_dict: Dictionary with the following keys.
file_dict.spc_dates_unix_sec: length-N numpy array of SPC dates.
file_dict.temp_file_names: 1-D list of paths to temp files (will be used for
intermediate IO).
file_dict.input_file_names_by_spc_date: length-N list, where the [i]th
element is a 1-D list (length T_i) of paths to input files.
file_dict.output_file_names_by_spc_date: Same but for output files.
:raises: ValueError: if start_time_unix_sec is not part of the first SPC
date (determined by start_spc_date_unix_sec).
:raises: ValueError: if end_time_unix_sec is not part of the last SPC date
(determined by end_spc_date_unix_sec).
"""
if not time_conversion.is_time_in_spc_date(start_time_unix_sec,
start_spc_date_string):
start_time_string = time_conversion.unix_sec_to_string(
start_time_unix_sec, TIME_FORMAT_FOR_MESSAGES)
raise ValueError('Start time (' + start_time_string +
') is not in first SPC date (' +
start_spc_date_string + ').')
if not time_conversion.is_time_in_spc_date(end_time_unix_sec,
end_spc_date_string):
end_time_string = time_conversion.unix_sec_to_string(
end_time_unix_sec, TIME_FORMAT_FOR_MESSAGES)
raise ValueError('End time (' + end_time_string +
') is not in last SPC date (' + end_spc_date_string +
').')
error_checking.assert_is_greater(end_time_unix_sec, start_time_unix_sec)
start_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
start_spc_date_string)
end_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
end_spc_date_string)
num_spc_dates = int(1 + (end_spc_date_unix_sec - start_spc_date_unix_sec) /
DAYS_TO_SECONDS)
spc_dates_unix_sec = numpy.linspace(start_spc_date_unix_sec,
end_spc_date_unix_sec,
num=num_spc_dates,
dtype=int)
temp_file_names = [''] * num_spc_dates
input_file_names_by_spc_date = [['']] * num_spc_dates
output_file_names_by_spc_date = [['']] * num_spc_dates
for i in range(num_spc_dates):
spc_dates_unix_sec[i] = time_conversion.time_to_spc_date_unix_sec(
spc_dates_unix_sec[i])
temp_file_names[i] = tempfile.NamedTemporaryFile(delete=False).name
input_file_names_by_spc_date[i] = (
tracking_io.find_processed_files_one_spc_date(
spc_dates_unix_sec[i],
data_source=data_source,
top_processed_dir_name=top_input_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
raise_error_if_missing=True))
this_num_files = len(input_file_names_by_spc_date[i])
these_times_unix_sec = numpy.full(this_num_files, -1, dtype=int)
output_file_names_by_spc_date[i] = [''] * this_num_files
for j in range(this_num_files):
these_times_unix_sec[j] = tracking_io.processed_file_name_to_time(
input_file_names_by_spc_date[i][j])
output_file_names_by_spc_date[i][j] = (
tracking_io.find_processed_file(
unix_time_sec=these_times_unix_sec[j],
data_source=data_source,
spc_date_unix_sec=spc_dates_unix_sec[i],
top_processed_dir_name=top_output_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
raise_error_if_missing=False))
if i == 0:
keep_time_indices = numpy.where(
these_times_unix_sec >= start_time_unix_sec)[0]
these_times_unix_sec = these_times_unix_sec[keep_time_indices]
input_file_names_by_spc_date[i] = [
input_file_names_by_spc_date[i][j] for j in keep_time_indices
]
output_file_names_by_spc_date[i] = [
output_file_names_by_spc_date[i][j] for j in keep_time_indices
]
if i == num_spc_dates - 1:
keep_time_indices = numpy.where(
these_times_unix_sec <= end_time_unix_sec)[0]
input_file_names_by_spc_date[i] = [
input_file_names_by_spc_date[i][j] for j in keep_time_indices
]
output_file_names_by_spc_date[i] = [
output_file_names_by_spc_date[i][j] for j in keep_time_indices
]
return {
SPC_DATES_KEY: spc_dates_unix_sec,
TEMP_FILE_NAMES_KEY: temp_file_names,
INPUT_FILE_NAMES_KEY: input_file_names_by_spc_date,
OUTPUT_FILE_NAMES_KEY: output_file_names_by_spc_date
}
speeds = []
for working_date_index in range(num_spc_dates):
date_in_memory_indices = utils._get_dates_needed(
working_date_index=working_date_index,
num_dates=num_spc_dates,
climatology_type=PASSAGE_CLIMATOLOGY_TYPE)
for i in range(num_spc_dates):
if i in date_in_memory_indices:
if storm_object_table_by_spc_date[i] is None:
# Find tracking files for [i]th date.
these_tracking_file_names = (
tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_strings[i],
data_source='segmotion',
top_processed_dir_name=TOP_PROCESSED_DIR_NAME,
tracking_scale_metres2=TRACKING_SCALE_METRES2))
# Read tracking files for [i]th date.
storm_object_table_by_spc_date[i] = (
tracking_io.read_many_processed_files(
these_tracking_file_names))
else:
print 'Clearing data for SPC date "{0:s}"...'.format(
spc_date_strings[i])
storm_object_table_by_spc_date[i] = None
print SEPARATOR_STRING
def _extract_storm_images(num_image_rows, num_image_columns, rotate_grids,
rotated_grid_spacing_metres, radar_field_names,
radar_heights_m_agl, spc_date_string,
top_radar_dir_name, top_tracking_dir_name,
tracking_scale_metres2, target_name,
top_target_dir_name, top_output_dir_name):
"""Extracts storm-centered radar images from GridRad data.
:param num_image_rows: See documentation at top of file.
:param num_image_columns: Same.
:param rotate_grids: Same.
:param rotated_grid_spacing_metres: Same.
:param radar_field_names: Same.
:param radar_heights_m_agl: Same.
:param spc_date_string: Same.
:param top_radar_dir_name: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param target_name: Same.
:param top_target_dir_name: Same.
:param top_output_dir_name: Same.
"""
if target_name in ['', 'None']:
target_name = None
if target_name is not None:
target_param_dict = target_val_utils.target_name_to_params(target_name)
target_file_name = target_val_utils.find_target_file(
top_directory_name=top_target_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_string)
print 'Reading data from: "{0:s}"...'.format(target_file_name)
target_dict = target_val_utils.read_target_values(
netcdf_file_name=target_file_name, target_name=target_name)
print '\n'
# Find storm objects on the given SPC date.
tracking_file_names = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
# Read storm objects on the given SPC date.
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)[storm_images.STORM_COLUMNS_NEEDED]
print SEPARATOR_STRING
if target_name is not None:
print(
'Removing storm objects without target values (variable = '
'"{0:s}")...').format(target_name)
these_indices = tracking_utils.find_storm_objects(
all_storm_ids=storm_object_table[
tracking_utils.STORM_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.TIME_COLUMN].values.astype(int),
storm_ids_to_keep=target_dict[target_val_utils.STORM_IDS_KEY],
times_to_keep_unix_sec=target_dict[
target_val_utils.VALID_TIMES_KEY],
allow_missing=False)
num_storm_objects_orig = len(storm_object_table.index)
storm_object_table = storm_object_table.iloc[these_indices]
num_storm_objects = len(storm_object_table.index)
print 'Removed {0:d} of {1:d} storm objects!\n'.format(
num_storm_objects_orig - num_storm_objects, num_storm_objects_orig)
# Extract storm-centered radar images.
storm_images.extract_storm_images_gridrad(
storm_object_table=storm_object_table,
top_radar_dir_name=top_radar_dir_name,
top_output_dir_name=top_output_dir_name,
num_storm_image_rows=num_image_rows,
num_storm_image_columns=num_image_columns,
rotate_grids=rotate_grids,
rotated_grid_spacing_metres=rotated_grid_spacing_metres,
radar_field_names=radar_field_names,
radar_heights_m_agl=radar_heights_m_agl)
def _find_io_files_for_renaming(top_input_dir_name, first_date_unix_sec,
last_date_unix_sec, top_output_dir_name):
"""Finds input and output files for renaming storms.
N = number of dates
:param top_input_dir_name: See documentation for `rename_storms.`
:param first_date_unix_sec: Same.
:param last_date_unix_sec: Same.
:param top_output_dir_name: Same.
:return: input_file_names_by_date: length-N list, where the [i]th item is a
numpy array of paths to input files for the [i]th date.
:return: output_file_names_by_date: Same as above, but for output files.
:return: valid_times_by_date_unix_sec: Same as above, but for valid times.
All 3 arrays for the [i]th date have the same length.
"""
dates_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_date_unix_sec,
end_time_unix_sec=last_date_unix_sec,
time_interval_sec=DAYS_TO_SECONDS,
include_endpoint=True)
date_strings = [
time_conversion.unix_sec_to_string(t, DATE_FORMAT)
for t in dates_unix_sec
]
num_dates = len(date_strings)
input_file_names_by_date = [numpy.array([], dtype=object)] * num_dates
output_file_names_by_date = [numpy.array([], dtype=object)] * num_dates
valid_times_by_date_unix_sec = [numpy.array([], dtype=int)] * num_dates
for i in range(num_dates):
print 'Finding input files for date {0:s}...'.format(date_strings[i])
(these_input_file_names,
_) = tracking_io.find_processed_files_one_spc_date(
spc_date_string=date_strings[i],
data_source=tracking_utils.PROBSEVERE_SOURCE_ID,
top_processed_dir_name=top_input_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
raise_error_if_missing=True)
these_input_file_names.sort()
these_valid_times_unix_sec = numpy.array([
tracking_io.processed_file_name_to_time(f)
for f in these_input_file_names
],
dtype=int)
these_output_file_names = []
for t in these_valid_times_unix_sec:
these_output_file_names.append(
tracking_io.find_processed_file(
unix_time_sec=t,
data_source=tracking_utils.PROBSEVERE_SOURCE_ID,
top_processed_dir_name=top_output_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
raise_error_if_missing=False))
input_file_names_by_date[i] = numpy.array(these_input_file_names,
dtype=object)
output_file_names_by_date[i] = numpy.array(these_output_file_names,
dtype=object)
valid_times_by_date_unix_sec[i] = these_valid_times_unix_sec
print SEPARATOR_STRING
return (input_file_names_by_date, output_file_names_by_date,
valid_times_by_date_unix_sec)
def _interp_soundings(spc_date_string, lead_times_seconds,
lag_time_for_convective_contamination_sec,
top_ruc_directory_name, top_rap_directory_name,
top_tracking_dir_name, tracking_scale_metres2,
top_output_dir_name):
"""Interpolates NWP sounding to each storm object at each lead time.
:param spc_date_string: See documentation at top of file.
:param lead_times_seconds: Same.
:param lag_time_for_convective_contamination_sec: Same.
:param top_ruc_directory_name: Same.
:param top_rap_directory_name: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param top_output_dir_name: Same.
:raises: ValueError: if model-initialization times needed are on opposite
sides of 0000 UTC 1 May 2012 (the cutoff between RUC and RAP models).
"""
lead_times_seconds = numpy.array(lead_times_seconds, dtype=int)
tracking_file_names, _ = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)
print SEPARATOR_STRING
first_storm_time_unix_sec = numpy.min(
storm_object_table[tracking_utils.TIME_COLUMN].values)
last_storm_time_unix_sec = numpy.max(
storm_object_table[tracking_utils.TIME_COLUMN].values)
first_init_time_unix_sec = number_rounding.floor_to_nearest(
(first_storm_time_unix_sec + numpy.min(lead_times_seconds) -
lag_time_for_convective_contamination_sec), HOURS_TO_SECONDS)
last_init_time_unix_sec = number_rounding.floor_to_nearest(
(last_storm_time_unix_sec + numpy.max(lead_times_seconds) -
lag_time_for_convective_contamination_sec), HOURS_TO_SECONDS)
extreme_init_times_unix_sec = numpy.array(
[first_init_time_unix_sec, last_init_time_unix_sec], dtype=int)
if numpy.all(extreme_init_times_unix_sec < FIRST_RAP_TIME_UNIX_SEC):
top_grib_directory_name = top_ruc_directory_name
model_name = nwp_model_utils.RUC_MODEL_NAME
elif numpy.all(extreme_init_times_unix_sec >= FIRST_RAP_TIME_UNIX_SEC):
top_grib_directory_name = top_rap_directory_name
model_name = nwp_model_utils.RAP_MODEL_NAME
else:
first_storm_time_string = time_conversion.unix_sec_to_string(
first_storm_time_unix_sec, STORM_TIME_FORMAT)
last_storm_time_string = time_conversion.unix_sec_to_string(
last_storm_time_unix_sec, STORM_TIME_FORMAT)
first_init_time_string = time_conversion.unix_sec_to_string(
first_init_time_unix_sec, MODEL_INIT_TIME_FORMAT)
last_init_time_string = time_conversion.unix_sec_to_string(
last_init_time_unix_sec, MODEL_INIT_TIME_FORMAT)
error_string = (
'First and last storm times are {0:s} and {1:s}. Thus, first and '
'last model-initialization times needed are {2:s} and {3:s}, which '
'are on opposite sides of {4:s} (the cutoff between RUC and RAP '
'models). The code is not generalized enough to interp data from '
'two different models. Sorry, eh?').format(
first_storm_time_string, last_storm_time_string,
first_init_time_string, last_init_time_string,
FIRST_RAP_TIME_STRING)
raise ValueError(error_string)
sounding_dict_by_lead_time = soundings.interp_soundings_to_storm_objects(
storm_object_table=storm_object_table,
top_grib_directory_name=top_grib_directory_name,
model_name=model_name,
use_all_grids=True,
height_levels_m_agl=soundings.DEFAULT_HEIGHT_LEVELS_M_AGL,
lead_times_seconds=lead_times_seconds,
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec,
wgrib_exe_name=WGRIB_EXE_NAME,
wgrib2_exe_name=WGRIB2_EXE_NAME,
raise_error_if_missing=False)
print SEPARATOR_STRING
num_lead_times = len(lead_times_seconds)
for k in range(num_lead_times):
this_sounding_file_name = soundings.find_sounding_file(
top_directory_name=top_output_dir_name,
spc_date_string=spc_date_string,
lead_time_seconds=lead_times_seconds[k],
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec,
raise_error_if_missing=False)
print 'Writing soundings to: "{0:s}"...'.format(
this_sounding_file_name)
soundings.write_soundings(
netcdf_file_name=this_sounding_file_name,
sounding_dict_height_coords=sounding_dict_by_lead_time[k],
lead_time_seconds=lead_times_seconds[k],
lag_time_for_convective_contamination_sec=
lag_time_for_convective_contamination_sec)
def _extract_storm_images(num_image_rows, num_image_columns, rotate_grids,
rotated_grid_spacing_metres, radar_field_names,
refl_heights_m_agl, spc_date_string,
tarred_myrorss_dir_name, untarred_myrorss_dir_name,
top_tracking_dir_name, tracking_scale_metres2,
target_name, top_target_dir_name,
top_output_dir_name):
"""Extracts storm-centered img for each field/height pair and storm object.
:param num_image_rows: See documentation at top of file.
:param num_image_columns: Same.
:param rotate_grids: Same.
:param rotated_grid_spacing_metres: Same.
:param radar_field_names: Same.
:param refl_heights_m_agl: Same.
:param spc_date_string: Same.
:param tarred_myrorss_dir_name: Same.
:param untarred_myrorss_dir_name: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param target_name: Same.
:param top_target_dir_name: Same.
:param top_output_dir_name: Same.
"""
if target_name in ['', 'None']:
target_name = None
if target_name is not None:
target_param_dict = target_val_utils.target_name_to_params(target_name)
target_file_name = target_val_utils.find_target_file(
top_directory_name=top_target_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_string)
print 'Reading data from: "{0:s}"...'.format(target_file_name)
target_dict = target_val_utils.read_target_values(
netcdf_file_name=target_file_name, target_name=target_name)
print '\n'
refl_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_utils.REFL_NAME)
# Untar files with azimuthal shear.
az_shear_field_names = list(
set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES))
if len(az_shear_field_names):
az_shear_tar_file_name = (
'{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar'.format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string))
myrorss_io.unzip_1day_tar_file(
tar_file_name=az_shear_tar_file_name,
field_names=az_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name)
print SEPARATOR_STRING
# Untar files with other radar fields.
non_shear_field_names = list(
set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES))
if len(non_shear_field_names):
non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=non_shear_tar_file_name,
field_names=non_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name,
refl_heights_m_asl=refl_heights_m_asl)
print SEPARATOR_STRING
# Read storm tracks for the given SPC date.
tracking_file_names = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
storm_object_table = tracking_io.read_many_processed_files(
tracking_file_names)[storm_images.STORM_COLUMNS_NEEDED]
print SEPARATOR_STRING
if target_name is not None:
print(
'Removing storm objects without target values (variable = '
'"{0:s}")...').format(target_name)
these_indices = tracking_utils.find_storm_objects(
all_storm_ids=storm_object_table[
tracking_utils.STORM_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.TIME_COLUMN].values.astype(int),
storm_ids_to_keep=target_dict[target_val_utils.STORM_IDS_KEY],
times_to_keep_unix_sec=target_dict[
target_val_utils.VALID_TIMES_KEY],
allow_missing=False)
num_storm_objects_orig = len(storm_object_table.index)
storm_object_table = storm_object_table.iloc[these_indices]
num_storm_objects = len(storm_object_table.index)
print 'Removed {0:d} of {1:d} storm objects!\n'.format(
num_storm_objects_orig - num_storm_objects, num_storm_objects_orig)
# Extract storm-centered radar images.
storm_images.extract_storm_images_myrorss_or_mrms(
storm_object_table=storm_object_table,
radar_source=radar_utils.MYRORSS_SOURCE_ID,
top_radar_dir_name=untarred_myrorss_dir_name,
top_output_dir_name=top_output_dir_name,
num_storm_image_rows=num_image_rows,
num_storm_image_columns=num_image_columns,
rotate_grids=rotate_grids,
rotated_grid_spacing_metres=rotated_grid_spacing_metres,
radar_field_names=radar_field_names,
reflectivity_heights_m_agl=refl_heights_m_agl)
print SEPARATOR_STRING
# Remove untarred MYRORSS files.
myrorss_io.remove_unzipped_data_1day(
spc_date_string=spc_date_string,
top_directory_name=untarred_myrorss_dir_name,
field_names=radar_field_names,
refl_heights_m_asl=refl_heights_m_asl)
print SEPARATOR_STRING
def _find_tracking_gaps(first_spc_date_string, last_spc_date_string,
top_tracking_dir_name, tracking_scale_metres2,
data_source, min_time_diff_seconds):
"""Finds gaps (temporal discontinuities) between storm-tracking files.
:param first_spc_date_string: See documentation at top of file.
:param last_spc_date_string: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param data_source: Same.
:param min_time_diff_seconds: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
tracking_file_names = []
unix_times_sec = numpy.array([], dtype=int)
num_spc_dates = len(spc_date_strings)
for i in range(num_spc_dates):
print 'Finding tracking files for SPC date "{0:s}"...'.format(
spc_date_strings[i])
these_file_names, _ = tracking_io.find_processed_files_one_spc_date(
spc_date_string=spc_date_strings[i],
data_source=data_source,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
raise_error_if_missing=False)
print len(these_file_names)
if not len(these_file_names):
continue
these_file_sizes_bytes = numpy.array(
[os.path.getsize(f) for f in these_file_names], dtype=int)
these_valid_indices = numpy.where(
these_file_sizes_bytes > FILE_SIZE_WITHOUT_STORMS_BYTES)[0]
these_file_names = [these_file_names[k] for k in these_valid_indices]
these_unix_times_sec = numpy.array([
tracking_io.processed_file_name_to_time(f)
for f in these_file_names
],
dtype=int)
these_sort_indices = numpy.argsort(these_unix_times_sec)
these_unix_times_sec = these_unix_times_sec[these_sort_indices]
these_file_names = [these_file_names[k] for k in these_sort_indices]
tracking_file_names += these_file_names
unix_times_sec = numpy.concatenate(
(unix_times_sec, these_unix_times_sec))
time_diffs_seconds = numpy.diff(unix_times_sec)
time_gap_indices = numpy.where(
time_diffs_seconds >= min_time_diff_seconds)[0]
num_time_gaps = len(time_gap_indices)
print(
'\nThere are {0:d} time gaps (successive files >= {1:d} seconds apart),'
' listed below:\n').format(num_time_gaps, min_time_diff_seconds)
for i in time_gap_indices:
this_start_time_string = time_conversion.unix_sec_to_string(
unix_times_sec[i], TIME_FORMAT)
this_end_time_string = time_conversion.unix_sec_to_string(
unix_times_sec[i + 1], TIME_FORMAT)
print 'Gap between {0:s} and {1:s} = {2:d} seconds'.format(
this_start_time_string, this_end_time_string,
time_diffs_seconds[i])
