def _convert_files(top_probsevere_dir_name, date_string, top_output_dir_name):
"""Converts probSevere tracking files for one day.
:param top_probsevere_dir_name: See documentation at top of file.
:param date_string: Same.
:param top_output_dir_name: Same.
"""
date_unix_sec = time_conversion.string_to_unix_sec(date_string,
DATE_FORMAT)
raw_file_names = probsevere_io.find_raw_files_one_day(
top_directory_name=top_probsevere_dir_name,
unix_time_sec=date_unix_sec,
file_extension=probsevere_io.ASCII_FILE_EXTENSION,
raise_error_if_all_missing=True)
for this_raw_file_name in raw_file_names:
print 'Reading data from "{0:s}"...'.format(this_raw_file_name)
this_storm_object_table = probsevere_io.read_raw_file(
this_raw_file_name)
this_time_unix_sec = probsevere_io.raw_file_name_to_time(
this_raw_file_name)
this_new_file_name = tracking_io.find_processed_file(
unix_time_sec=this_time_unix_sec,
data_source=tracking_utils.PROBSEVERE_SOURCE_ID,
top_processed_dir_name=top_output_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
raise_error_if_missing=False)
print 'Writing data to "{0:s}"...'.format(this_new_file_name)
tracking_io.write_processed_file(
storm_object_table=this_storm_object_table,
pickle_file_name=this_new_file_name)
def _shuffle_io_for_renaming(input_file_names_by_date,
output_file_names_by_date,
valid_times_by_date_unix_sec,
storm_object_table_by_date, working_date_index):
"""Shuffles data into and out of memory for renaming storms.
For more on renaming storms, see doc for `rename_storms`.
N = number of dates
:param input_file_names_by_date: length-N list created by
`_find_io_files_for_renaming`.
:param output_file_names_by_date: Same.
:param valid_times_by_date_unix_sec: Same.
:param storm_object_table_by_date: length-N list, where the [i]th element is
a pandas DataFrame with tracking data for the [i]th date. At any given
time, all but 3 items should be None. Each table has columns documented
in `storm_tracking_io.write_processed_file`, plus the following column.
storm_object_table_by_date.date_index: Array index. If date_index[i] = j,
the [i]th row (storm object) comes from the [j]th date.
:param working_date_index: Index of date currently being processed. Only
dates (working_date_index - 1)...(working_date_index + 1) need to be in
memory. If None, this method will write/clear all data currently in
memory, without reading new data.
:return: storm_object_table_by_date: Same as input, except that different
items are filled and different items are None.
"""
num_dates = len(input_file_names_by_date)
if working_date_index is None:
date_needed_indices = numpy.array([-1], dtype=int)
else:
date_needed_indices = _get_dates_needed_for_renaming_storms(
working_date_index=working_date_index,
num_dates_in_period=num_dates)
for j in range(num_dates):
if j in date_needed_indices and storm_object_table_by_date[j] is None:
storm_object_table_by_date[j] = (
tracking_io.read_many_processed_files(
input_file_names_by_date[j].tolist()))
print SEPARATOR_STRING
this_num_storm_objects = len(storm_object_table_by_date[j].index)
these_date_indices = numpy.full(this_num_storm_objects,
j,
dtype=int)
argument_dict = {DATE_INDEX_KEY: these_date_indices}
storm_object_table_by_date[j] = storm_object_table_by_date[
j].assign(**argument_dict)
if j not in date_needed_indices:
if storm_object_table_by_date[j] is not None:
these_output_file_names = output_file_names_by_date[j]
these_valid_times_unix_sec = valid_times_by_date_unix_sec[j]
for k in range(len(these_valid_times_unix_sec)):
print 'Writing new data to "{0:s}"...'.format(
these_output_file_names[k])
these_indices = numpy.where(
storm_object_table_by_date[j][
tracking_utils.TIME_COLUMN].values ==
these_valid_times_unix_sec[k])[0]
tracking_io.write_processed_file(
storm_object_table=storm_object_table_by_date[j].
iloc[these_indices],
pickle_file_name=these_output_file_names[k])
print SEPARATOR_STRING
storm_object_table_by_date[j] = None
if working_date_index is None:
return storm_object_table_by_date
for j in date_needed_indices[1:]:
storm_object_table_by_date[j], _ = storm_object_table_by_date[j].align(
storm_object_table_by_date[j - 1], axis=1)
return storm_object_table_by_date
NETCDF_FILE_NAME, data_source=radar_io.MYRORSS_SOURCE_ID)
POLYGON_TABLE = read_polygons_from_netcdf(
NETCDF_FILE_NAME,
metadata_dict=METADATA_DICT,
spc_date_unix_sec=SPC_DATE_UNIX_SEC,
tracking_start_time_unix_sec=TRACKING_START_TIME_UNIX_SEC,
tracking_end_time_unix_sec=TRACKING_END_TIME_UNIX_SEC)
print POLYGON_TABLE
POLYGON_TABLE = tracking_io.make_buffers_around_polygons(
POLYGON_TABLE,
min_buffer_dists_metres=MIN_BUFFER_DISTS_METRES,
max_buffer_dists_metres=MAX_BUFFER_DISTS_METRES)
print POLYGON_TABLE
STORM_TABLE = join_stats_and_polygons(STATS_TABLE, POLYGON_TABLE)
print STORM_TABLE
VALID_TIME_UNIX_SEC = time_conversion.string_to_unix_sec(
VALID_TIME_STRING, VALID_TIME_FORMAT)
OUTPUT_FILE_NAME = tracking_io.find_processed_file(
unix_time_sec=VALID_TIME_UNIX_SEC,
data_source=tracking_io.SEGMOTION_SOURCE_ID,
spc_date_unix_sec=SPC_DATE_UNIX_SEC,
top_processed_dir_name=TOP_PROCESSED_DIR_NAME,
tracking_scale_metres2=TRACKING_SCALE_METRES2,
raise_error_if_missing=False)
tracking_io.write_processed_file(STORM_TABLE, OUTPUT_FILE_NAME)
storm_object_table[
tracking_io.POLYGON_OBJECT_ROWCOL_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(
these_vertex_columns, these_vertex_rows))
storm_object_table[
tracking_io.POLYGON_OBJECT_LATLNG_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(
these_vertex_lng_deg, these_vertex_lat_deg))
return storm_object_table
if __name__ == '__main__':
STORM_OBJECT_TABLE = read_storm_objects_from_raw_file(RAW_FILE_NAME)
print STORM_OBJECT_TABLE
STORM_OBJECT_TABLE = tracking_io.make_buffers_around_polygons(
STORM_OBJECT_TABLE, min_buffer_dists_metres=MIN_BUFFER_DISTS_METRES,
max_buffer_dists_metres=MAX_BUFFER_DISTS_METRES)
print STORM_OBJECT_TABLE
PROCESSED_FILE_NAME = tracking_io.find_processed_file(
unix_time_sec=UNIX_TIME_SEC,
data_source=tracking_io.PROBSEVERE_SOURCE_ID,
top_processed_dir_name=TOP_PROCESSED_DIR_NAME,
tracking_scale_metres2=TRACKING_SCALE_METRES2,
raise_error_if_missing=False)
tracking_io.write_processed_file(STORM_OBJECT_TABLE, PROCESSED_FILE_NAME)