def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string,
top_myrorss_dir_name, radar_field_name, output_file_name):
"""Evaluates a set of storm tracks.
This is effectively the main method.
:param top_tracking_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param top_myrorss_dir_name: Same.
:param radar_field_name: Same.
:param output_file_name: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
list_of_storm_object_tables = []
for this_spc_date_string in spc_date_strings:
these_file_names = tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=echo_top_tracking.
DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
if len(these_file_names) == 0:
continue
this_storm_object_table = tracking_io.read_many_files(
these_file_names)[STORM_OBJECT_COLUMNS]
list_of_storm_object_tables.append(this_storm_object_table)
if this_spc_date_string != spc_date_strings[-1]:
print(MINOR_SEPARATOR_STRING)
if len(list_of_storm_object_tables) == 1:
continue
list_of_storm_object_tables[-1] = list_of_storm_object_tables[
-1].align(list_of_storm_object_tables[0], axis=1)[0]
print(SEPARATOR_STRING)
storm_object_table = pandas.concat(list_of_storm_object_tables,
axis=0,
ignore_index=True)
evaluation_dict = tracking_eval.evaluate_tracks(
storm_object_table=storm_object_table,
top_myrorss_dir_name=top_myrorss_dir_name,
radar_field_name=radar_field_name)
print('Writing results to: "{0:s}"...'.format(output_file_name))
tracking_eval.write_file(evaluation_dict=evaluation_dict,
pickle_file_name=output_file_name)
def _run(tornado_dir_name, top_gridrad_dir_name, first_spc_date_string,
last_spc_date_string, output_dir_name):
"""Plots histograms for GridRad dataset.
This is effectively the main method.
:param tornado_dir_name: See documentation at top of file.
:param top_gridrad_dir_name: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param output_dir_name: Same.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
all_spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
spc_date_strings = []
for this_spc_date_string in all_spc_date_strings:
this_gridrad_file_name = _find_gridrad_file_for_date(
top_gridrad_dir_name=top_gridrad_dir_name,
spc_date_string=this_spc_date_string)
if this_gridrad_file_name is None:
continue
spc_date_strings.append(this_spc_date_string)
first_year, last_year = _spc_dates_to_years(spc_date_strings)
tornado_table = _read_tornado_reports(tornado_dir_name=tornado_dir_name,
first_year=first_year,
last_year=last_year)
print(SEPARATOR_STRING)
num_days = len(spc_date_strings)
num_tornadoes_by_day = numpy.full(num_days, -1, dtype=int)
for i in range(num_days):
num_tornadoes_by_day[i] = _get_num_tornadoes_in_day(
tornado_table=tornado_table, spc_date_string=spc_date_strings[i])
print('Number of tornadoes on SPC date "{0:s}" = {1:d}'.format(
spc_date_strings[i], num_tornadoes_by_day[i]))
print(SEPARATOR_STRING)
_plot_tornado_histogram(
num_tornadoes_by_day=num_tornadoes_by_day,
output_file_name='{0:s}/tornado_histogram.jpg'.format(output_dir_name))
_plot_month_histogram(
spc_date_strings=spc_date_strings,
output_file_name='{0:s}/month_histogram.jpg'.format(output_dir_name))
def _run(echo_top_field_name, top_radar_dir_name_tarred, top_radar_dir_name,
top_echo_classifn_dir_name, min_echo_top_km_asl,
min_grid_cells_in_polygon, top_output_dir_name, first_spc_date_string,
last_spc_date_string):
"""Runs echo-top-based storm-tracking.
This is effectively the main method.
:param echo_top_field_name: See documentation at top of file.
:param top_radar_dir_name_tarred: Same.
:param top_radar_dir_name: Same.
:param top_echo_classifn_dir_name: Same.
:param min_echo_top_km_asl: Same.
:param min_grid_cells_in_polygon: Same.
:param top_output_dir_name: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
"""
if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES:
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
for this_spc_date_string in spc_date_strings:
this_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format(
top_radar_dir_name_tarred, this_spc_date_string[:4],
this_spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=this_tar_file_name,
field_names=[echo_top_field_name],
spc_date_string=this_spc_date_string,
top_target_directory_name=top_radar_dir_name)
print SEPARATOR_STRING
if top_echo_classifn_dir_name in ['', 'None']:
top_echo_classifn_dir_name = None
echo_top_tracking.run_tracking(
top_radar_dir_name=top_radar_dir_name,
top_output_dir_name=top_output_dir_name,
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string,
echo_top_field_name=echo_top_field_name,
top_echo_classifn_dir_name=top_echo_classifn_dir_name,
min_echo_top_height_km_asl=min_echo_top_km_asl,
min_grid_cells_in_polygon=min_grid_cells_in_polygon)
print SEPARATOR_STRING
if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES:
for this_spc_date_string in spc_date_strings:
myrorss_io.remove_unzipped_data_1day(
spc_date_string=this_spc_date_string,
top_directory_name=top_radar_dir_name,
field_names=[echo_top_field_name])
def test_get_spc_dates_in_range_many_dates(self):
"""Ensures correct output from get_spc_dates_in_range.
In this case, there are many dates in the range.
"""
these_spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=FIRST_SPC_DATE_STRING,
last_spc_date_string=LAST_SPC_DATE_STRING)
self.assertTrue(these_spc_date_strings == ALL_SPC_DATE_STRINGS)
def test_get_spc_dates_in_range_one_date(self):
"""Ensures correct output from get_spc_dates_in_range.
In this case, there is only one date in the range.
"""
these_spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=FIRST_SPC_DATE_STRING,
last_spc_date_string=FIRST_SPC_DATE_STRING)
self.assertTrue(these_spc_date_strings == [FIRST_SPC_DATE_STRING])
def _run(top_input_dir_name, radar_field_names, refl_heights_m_asl,
first_spc_date_string, last_spc_date_string, top_output_dir_name):
"""Untars MYRORSS data.
This is effectively the main method.
:param top_input_dir_name: See documentation at top of file.
:param radar_field_names: Same.
:param refl_heights_m_asl: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param top_output_dir_name: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
az_shear_field_names = list(
set(radar_field_names) & set(radar_utils.SHEAR_NAMES))
non_shear_field_names = list(
set(radar_field_names) - set(radar_utils.SHEAR_NAMES))
for this_spc_date_string in spc_date_strings:
if len(az_shear_field_names) > 0:
this_tar_file_name = (
'{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar').format(
top_input_dir_name, this_spc_date_string[:4],
this_spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=this_tar_file_name,
field_names=az_shear_field_names,
spc_date_string=this_spc_date_string,
top_target_directory_name=top_output_dir_name)
print(SEPARATOR_STRING)
if len(non_shear_field_names) > 0:
this_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format(
top_input_dir_name, this_spc_date_string[:4],
this_spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=this_tar_file_name,
field_names=non_shear_field_names,
spc_date_string=this_spc_date_string,
top_target_directory_name=top_output_dir_name,
refl_heights_m_asl=refl_heights_m_asl)
print(SEPARATOR_STRING)
def _run(tornado_dir_name, top_tracking_dir_name, tracking_scale_metres2,
genesis_only, first_spc_date_string, last_spc_date_string,
top_output_dir_name):
"""Runs `linkage.link_tornadoes_to_storms`.
This is effectively the main method.
:param tornado_dir_name: See documentation at top of file.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param genesis_only: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param top_output_dir_name: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
tracking_file_names = []
for this_spc_date_string in spc_date_strings:
these_file_names = tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
if len(these_file_names) == 0:
if len(tracking_file_names) > 0:
_link_tornadoes_one_period(
tracking_file_names=tracking_file_names,
tornado_dir_name=tornado_dir_name,
genesis_only=genesis_only,
top_output_dir_name=top_output_dir_name)
print(SEPARATOR_STRING)
tracking_file_names = []
continue
tracking_file_names += these_file_names
_link_tornadoes_one_period(tracking_file_names=tracking_file_names,
tornado_dir_name=tornado_dir_name,
genesis_only=genesis_only,
top_output_dir_name=top_output_dir_name)
def _write_csv_file_for_thea(
first_spc_date_string, last_spc_date_string, top_tracking_dir_name,
tracking_scale_metres2, output_file_name):
"""Converts storm tracks to a single CSV file for Thea.
:param first_spc_date_string: See documentation at top of file.
:param last_spc_date_string: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param output_file_name: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
tracking_file_names = []
for this_spc_date_string in spc_date_strings:
(these_tracking_file_names, _
) = tracking_io.find_processed_files_one_spc_date(
spc_date_string=this_spc_date_string,
data_source=tracking_utils.SEGMOTION_SOURCE_ID,
top_processed_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)
tracking_file_names += these_tracking_file_names
num_files = len(tracking_file_names)
list_of_storm_object_tables = [None] * num_files
for i in range(num_files):
print 'Reading {0:d}th of {1:d} input files: "{2:s}"...'.format(
i + 1, num_files, tracking_file_names[i])
list_of_storm_object_tables[i] = tracking_io.read_processed_file(
tracking_file_names[i])
if i == 0:
continue
list_of_storm_object_tables[i], _ = (
list_of_storm_object_tables[i].align(
list_of_storm_object_tables[0], axis=1))
storm_object_table = pandas.concat(
list_of_storm_object_tables, axis=0, ignore_index=True)
print 'Writing all storm tracks to "{0:s}"...'.format(output_file_name)
best_tracks.write_simple_output_for_thea(
storm_object_table, output_file_name)
def _run(top_input_dir_name, first_spc_date_string, last_spc_date_string,
resolution_factor, top_output_dir_name):
"""Converts examples from GridRad to MYRORSS format.
This is effectively the main method.
:param top_input_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param resolution_factor: Same.
:param top_output_dir_name: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
input_file_names = [
input_examples.find_example_file(
top_directory_name=top_input_dir_name, shuffled=False,
spc_date_string=d, raise_error_if_missing=False
)
for d in spc_date_strings
]
output_file_names = [
input_examples.find_example_file(
top_directory_name=top_output_dir_name, shuffled=False,
spc_date_string=d, raise_error_if_missing=False
)
for d in spc_date_strings
]
num_spc_dates = len(spc_date_strings)
for i in range(num_spc_dates):
if not os.path.isfile(input_file_names[i]):
continue
_convert_one_file(
input_file_name=input_file_names[i],
resolution_factor=resolution_factor,
output_file_name=output_file_names[i]
)
print('\n')
def _run(top_orig_tracking_dir_name, top_new_tracking_dir_name,
first_spc_date_string, last_spc_date_string, output_file_name):
"""Plots storms that were removed by remove_storms_outside_conus.py.
This is effectively the main method.
:param top_orig_tracking_dir_name: See documentation at top of file.
:param top_new_tracking_dir_name: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param output_file_name: Same.
"""
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
orig_tracking_file_names = []
for d in spc_date_strings:
orig_tracking_file_names += tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_orig_tracking_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=d,
raise_error_if_missing=False)[0]
valid_times_unix_sec = numpy.array(
[tracking_io.file_name_to_time(f) for f in orig_tracking_file_names],
dtype=int)
new_tracking_file_names = [
tracking_io.find_file(
top_tracking_dir_name=top_new_tracking_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
valid_time_unix_sec=t,
spc_date_string=time_conversion.time_to_spc_date_string(t),
raise_error_if_missing=True) for t in valid_times_unix_sec
]
orig_storm_object_table = tracking_io.read_many_files(
orig_tracking_file_names)
print(SEPARATOR_STRING)
new_storm_object_table = tracking_io.read_many_files(
new_tracking_file_names)
print(SEPARATOR_STRING)
orig_storm_id_strings = (
orig_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist())
orig_storm_times_unix_sec = (
orig_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
new_storm_id_strings = (
new_storm_object_table[tracking_utils.FULL_ID_COLUMN].values.tolist())
new_storm_times_unix_sec = (
new_storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
num_orig_storm_objects = len(orig_storm_object_table.index)
orig_kept_flags = numpy.full(num_orig_storm_objects, 0, dtype=bool)
these_indices = tracking_utils.find_storm_objects(
all_id_strings=orig_storm_id_strings,
all_times_unix_sec=orig_storm_times_unix_sec,
id_strings_to_keep=new_storm_id_strings,
times_to_keep_unix_sec=new_storm_times_unix_sec,
allow_missing=False)
orig_kept_flags[these_indices] = True
orig_removed_indices = numpy.where(numpy.invert(orig_kept_flags))[0]
print('{0:d} of {1:d} storm objects were outside CONUS.'.format(
len(orig_removed_indices), num_orig_storm_objects))
removed_storm_object_table = orig_storm_object_table.iloc[
orig_removed_indices]
removed_latitudes_deg = removed_storm_object_table[
tracking_utils.CENTROID_LATITUDE_COLUMN].values
removed_longitudes_deg = removed_storm_object_table[
tracking_utils.CENTROID_LONGITUDE_COLUMN].values
figure_object, axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=numpy.min(removed_latitudes_deg) - 1.,
max_latitude_deg=numpy.max(removed_latitudes_deg) + 1.,
min_longitude_deg=numpy.min(removed_longitudes_deg) - 1.,
max_longitude_deg=numpy.max(removed_longitudes_deg) + 1.,
resolution_string='i'))
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS)
conus_latitudes_deg, conus_longitudes_deg = (
conus_boundary.read_from_netcdf())
conus_latitudes_deg, conus_longitudes_deg = conus_boundary.erode_boundary(
latitudes_deg=conus_latitudes_deg,
longitudes_deg=conus_longitudes_deg,
erosion_distance_metres=EROSION_DISTANCE_METRES)
axes_object.plot(conus_longitudes_deg,
conus_latitudes_deg,
color=LINE_COLOUR,
linestyle='solid',
linewidth=LINE_WIDTH)
axes_object.plot(removed_longitudes_deg,
removed_latitudes_deg,
linestyle='None',
marker=MARKER_TYPE,
markersize=MARKER_SIZE,
markeredgewidth=0,
markerfacecolor=MARKER_COLOUR,
markeredgecolor=MARKER_COLOUR)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
figure_object.savefig(output_file_name,
dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0,
bbox_inches='tight')
pyplot.close(figure_object)
def _run(top_gridrad_dir_name, first_spc_date_string, last_spc_date_string):
"""Makes LaTeX table with GridRad days and domains.
This is effectively the main method.
:param top_gridrad_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
"""
all_spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
nice_date_strings = []
latitude_strings = []
longitude_strings = []
for this_spc_date_string in all_spc_date_strings:
these_limits_deg = _find_domain_for_date(
top_gridrad_dir_name=top_gridrad_dir_name,
spc_date_string=this_spc_date_string
)
if these_limits_deg is None:
continue
print(SEPARATOR_STRING)
this_time_unix_sec = time_conversion.spc_date_string_to_unix_sec(
this_spc_date_string
)
this_nice_date_string = time_conversion.unix_sec_to_string(
this_time_unix_sec, NICE_TIME_FORMAT
)
nice_date_strings.append(this_nice_date_string)
latitude_strings.append('{0:.1f}-{1:.1f}'.format(
these_limits_deg[0], these_limits_deg[1]
))
longitude_strings.append('{0:.1f}-{1:.1f}'.format(
these_limits_deg[3], these_limits_deg[2]
))
table_string = ''
for i in range(len(nice_date_strings)):
if i != 0:
if numpy.mod(i, 2) == 0:
table_string += ' \\\\\n\t\t\t'
else:
table_string += ' & '
table_string += '{0:s}, {1:s}'.format(
nice_date_strings[i], latitude_strings[i]
)
# table_string += ' $^{\\circ}$N'
table_string += ', {0:s}'.format(longitude_strings[i])
# table_string += ' $^{\\circ}$W'
print(table_string)
def _run(top_input_dir_name, target_name, first_spc_date_string,
last_spc_date_string, class_fraction_keys, class_fraction_values,
for_training, top_output_dir_name):
"""Downsamples storm objects, based on target values.
This is effectively the main method.
:param top_input_dir_name: See documentation at top of file.
:param target_name: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param class_fraction_keys: Same.
:param class_fraction_values: Same.
:param for_training: Same.
:param top_output_dir_name: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
class_fraction_dict = dict(zip(class_fraction_keys, class_fraction_values))
target_param_dict = target_val_utils.target_name_to_params(target_name)
input_target_file_names = []
spc_date_string_by_file = []
for this_spc_date_string in spc_date_strings:
this_file_name = target_val_utils.find_target_file(
top_directory_name=top_input_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)
if not os.path.isfile(this_file_name):
continue
input_target_file_names.append(this_file_name)
spc_date_string_by_file.append(this_spc_date_string)
num_files = len(input_target_file_names)
spc_date_strings = spc_date_string_by_file
target_dict_by_file = [None] * num_files
storm_ids = []
storm_times_unix_sec = numpy.array([], dtype=int)
target_values = numpy.array([], dtype=int)
for i in range(num_files):
print 'Reading "{0:s}" from: "{1:s}"...'.format(
target_name, input_target_file_names[i])
target_dict_by_file[i] = target_val_utils.read_target_values(
netcdf_file_name=input_target_file_names[i],
target_name=target_name)
storm_ids += target_dict_by_file[i][target_val_utils.STORM_IDS_KEY]
storm_times_unix_sec = numpy.concatenate(
(storm_times_unix_sec,
target_dict_by_file[i][target_val_utils.VALID_TIMES_KEY]))
target_values = numpy.concatenate(
(target_values,
target_dict_by_file[i][target_val_utils.TARGET_VALUES_KEY]))
print SEPARATOR_STRING
good_indices = numpy.where(
target_values != target_val_utils.INVALID_STORM_INTEGER)[0]
storm_ids = [storm_ids[k] for k in good_indices]
storm_times_unix_sec = storm_times_unix_sec[good_indices]
target_values = target_values[good_indices]
if for_training:
storm_ids, storm_times_unix_sec, target_values = (
fancy_downsampling.downsample_for_training(
storm_ids=storm_ids,
storm_times_unix_sec=storm_times_unix_sec,
target_values=target_values,
target_name=target_name,
class_fraction_dict=class_fraction_dict))
else:
storm_ids, storm_times_unix_sec, target_values = (
fancy_downsampling.downsample_for_non_training(
storm_ids=storm_ids,
storm_times_unix_sec=storm_times_unix_sec,
target_values=target_values,
target_name=target_name,
class_fraction_dict=class_fraction_dict))
print SEPARATOR_STRING
for i in range(num_files):
these_indices = tracking_utils.find_storm_objects(
all_storm_ids=target_dict_by_file[i][
target_val_utils.STORM_IDS_KEY],
all_times_unix_sec=target_dict_by_file[i][
target_val_utils.VALID_TIMES_KEY],
storm_ids_to_keep=storm_ids,
times_to_keep_unix_sec=storm_times_unix_sec,
allow_missing=True)
these_indices = these_indices[these_indices >= 0]
if len(these_indices) == 0:
continue
this_output_dict = {
tracking_utils.STORM_ID_COLUMN: [
target_dict_by_file[i][target_val_utils.STORM_IDS_KEY][k]
for k in these_indices
],
tracking_utils.TIME_COLUMN:
target_dict_by_file[i][
target_val_utils.VALID_TIMES_KEY][these_indices],
target_name:
target_dict_by_file[i][target_val_utils.TARGET_VALUES_KEY]
[these_indices]
}
this_output_table = pandas.DataFrame.from_dict(this_output_dict)
this_new_file_name = target_val_utils.find_target_file(
top_directory_name=top_output_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_strings[i],
raise_error_if_missing=False)
print(
'Writing {0:d} downsampled storm objects (out of {1:d} total) to: '
'"{2:s}"...').format(
len(this_output_table.index),
len(target_dict_by_file[i][target_val_utils.STORM_IDS_KEY]),
this_new_file_name)
target_val_utils.write_target_values(
storm_to_events_table=this_output_table,
target_names=[target_name],
netcdf_file_name=this_new_file_name)
def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string,
colour_map_name, min_plot_latitude_deg, max_plot_latitude_deg,
min_plot_longitude_deg, max_plot_longitude_deg, output_file_name):
"""Plots storm tracks for a continuous time period.
This is effectively the main method.
:param top_tracking_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param colour_map_name: Same.
:param min_plot_latitude_deg: Same.
:param max_plot_latitude_deg: Same.
:param min_plot_longitude_deg: Same.
:param max_plot_longitude_deg: Same.
:param output_file_name: Same.
"""
if colour_map_name in ['', 'None']:
colour_map_object = 'random'
else:
colour_map_object = pyplot.cm.get_cmap(colour_map_name)
if min_plot_latitude_deg <= SENTINEL_VALUE:
min_plot_latitude_deg = None
if max_plot_latitude_deg <= SENTINEL_VALUE:
max_plot_latitude_deg = None
if min_plot_longitude_deg <= SENTINEL_VALUE:
min_plot_longitude_deg = None
if max_plot_longitude_deg <= SENTINEL_VALUE:
max_plot_longitude_deg = None
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
list_of_storm_object_tables = []
for this_spc_date_string in spc_date_strings:
these_file_names = tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=echo_top_tracking.
DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
if len(these_file_names) == 0:
continue
this_storm_object_table = tracking_io.read_many_files(
these_file_names)[REQUIRED_COLUMNS]
list_of_storm_object_tables.append(this_storm_object_table)
if this_spc_date_string != spc_date_strings[-1]:
print(MINOR_SEPARATOR_STRING)
if len(list_of_storm_object_tables) == 1:
continue
list_of_storm_object_tables[-1] = list_of_storm_object_tables[
-1].align(list_of_storm_object_tables[0], axis=1)[0]
print(SEPARATOR_STRING)
storm_object_table = pandas.concat(list_of_storm_object_tables,
axis=0,
ignore_index=True)
if min_plot_latitude_deg is None:
min_plot_latitude_deg = numpy.min(
storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values
) - LATLNG_BUFFER_DEG
if max_plot_latitude_deg is None:
max_plot_latitude_deg = numpy.max(
storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values
) + LATLNG_BUFFER_DEG
if min_plot_longitude_deg is None:
min_plot_longitude_deg = numpy.min(
storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values
) - LATLNG_BUFFER_DEG
if max_plot_longitude_deg is None:
max_plot_longitude_deg = numpy.max(
storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values
) + LATLNG_BUFFER_DEG
_, axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg,
resolution_string='i'))
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS)
storm_plotting.plot_storm_tracks(storm_object_table=storm_object_table,
axes_object=axes_object,
basemap_object=basemap_object,
colour_map_object=colour_map_object)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string,
colour_map_name, min_plot_latitude_deg, max_plot_latitude_deg,
min_plot_longitude_deg, max_plot_longitude_deg, output_file_name):
"""Plots storm tracks for a continuous time period.
This is effectively the main method.
:param top_tracking_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param colour_map_name: Same.
:param min_plot_latitude_deg: Same.
:param max_plot_latitude_deg: Same.
:param min_plot_longitude_deg: Same.
:param max_plot_longitude_deg: Same.
:param output_file_name: Same.
"""
if colour_map_name in ['', 'None']:
colour_map_object = 'random'
else:
colour_map_object = pyplot.cm.get_cmap(colour_map_name)
if min_plot_latitude_deg <= SENTINEL_VALUE:
min_plot_latitude_deg = None
if max_plot_latitude_deg <= SENTINEL_VALUE:
max_plot_latitude_deg = None
if min_plot_longitude_deg <= SENTINEL_VALUE:
min_plot_longitude_deg = None
if max_plot_longitude_deg <= SENTINEL_VALUE:
max_plot_longitude_deg = None
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
list_of_storm_object_tables = []
for this_spc_date_string in spc_date_strings:
these_file_names = tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=echo_top_tracking.
DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
if len(these_file_names) == 0:
continue
this_storm_object_table = tracking_io.read_many_files(
these_file_names)[REQUIRED_COLUMNS]
list_of_storm_object_tables.append(this_storm_object_table)
if this_spc_date_string != spc_date_strings[-1]:
print(MINOR_SEPARATOR_STRING)
if len(list_of_storm_object_tables) == 1:
continue
list_of_storm_object_tables[-1] = list_of_storm_object_tables[
-1].align(list_of_storm_object_tables[0], axis=1)[0]
print(SEPARATOR_STRING)
storm_object_table = pandas.concat(list_of_storm_object_tables,
axis=0,
ignore_index=True)
# TODO(thunderhoser): HACK
first_time_unix_sec = time_conversion.string_to_unix_sec(
'2011-04-27-20', '%Y-%m-%d-%H')
storm_object_table = storm_object_table.loc[storm_object_table[
tracking_utils.VALID_TIME_COLUMN] >= first_time_unix_sec]
if min_plot_latitude_deg is None:
min_plot_latitude_deg = numpy.min(
storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values
) - LATLNG_BUFFER_DEG
if max_plot_latitude_deg is None:
max_plot_latitude_deg = numpy.max(
storm_object_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values
) + LATLNG_BUFFER_DEG
if min_plot_longitude_deg is None:
min_plot_longitude_deg = numpy.min(
storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values
) - LATLNG_BUFFER_DEG
if max_plot_longitude_deg is None:
max_plot_longitude_deg = numpy.max(
storm_object_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values
) + LATLNG_BUFFER_DEG
_, axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg,
resolution_string='i'))
# plotting_utils.plot_coastlines(
# basemap_object=basemap_object, axes_object=axes_object,
# line_colour=BORDER_COLOUR
# )
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS,
line_colour=numpy.full(3, 1.))
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS,
line_colour=numpy.full(3, 1.))
colour_bar_object = storm_plotting.plot_storm_tracks(
storm_object_table=storm_object_table,
axes_object=axes_object,
basemap_object=basemap_object,
colour_map_object=colour_map_object)
valid_times_unix_sec = (
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
# TODO(thunderhoser): HACK
tick_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=numpy.min(valid_times_unix_sec),
end_time_unix_sec=numpy.max(valid_times_unix_sec),
time_interval_sec=1800,
include_endpoint=True)
tick_time_strings = [
time_conversion.unix_sec_to_string(t, COLOUR_BAR_TIME_FORMAT)
for t in tick_times_unix_sec
]
colour_bar_object.set_ticks(tick_times_unix_sec)
colour_bar_object.set_ticklabels(tick_time_strings)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
pyplot.savefig(output_file_name,
dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0,
bbox_inches='tight')
pyplot.close()
def _run(top_tracking_dir_name, first_spc_date_string, last_spc_date_string,
storm_colour, storm_opacity, include_secondary_ids,
min_plot_latitude_deg, max_plot_latitude_deg, min_plot_longitude_deg,
max_plot_longitude_deg, top_myrorss_dir_name, radar_field_name,
radar_height_m_asl, output_dir_name):
"""Plots storm outlines (along with IDs) at each time step.
This is effectively the main method.
:param top_tracking_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param storm_colour: Same.
:param storm_opacity: Same.
:param include_secondary_ids: Same.
:param min_plot_latitude_deg: Same.
:param max_plot_latitude_deg: Same.
:param min_plot_longitude_deg: Same.
:param max_plot_longitude_deg: Same.
:param top_myrorss_dir_name: Same.
:param radar_field_name: Same.
:param radar_height_m_asl: Same.
:param output_dir_name: Same.
"""
if top_myrorss_dir_name in ['', 'None']:
top_myrorss_dir_name = None
if radar_field_name != radar_utils.REFL_NAME:
radar_height_m_asl = None
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
tracking_file_names = []
for this_spc_date_string in spc_date_strings:
tracking_file_names += (tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=DUMMY_TRACKING_SCALE_METRES2,
source_name=DUMMY_SOURCE_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0])
storm_object_table = tracking_io.read_many_files(tracking_file_names)
print(SEPARATOR_STRING)
latitude_limits_deg, longitude_limits_deg = _get_plotting_limits(
min_plot_latitude_deg=min_plot_latitude_deg,
max_plot_latitude_deg=max_plot_latitude_deg,
min_plot_longitude_deg=min_plot_longitude_deg,
max_plot_longitude_deg=max_plot_longitude_deg,
storm_object_table=storm_object_table)
min_plot_latitude_deg = latitude_limits_deg[0]
max_plot_latitude_deg = latitude_limits_deg[1]
min_plot_longitude_deg = longitude_limits_deg[0]
max_plot_longitude_deg = longitude_limits_deg[1]
valid_times_unix_sec = numpy.unique(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
num_times = len(valid_times_unix_sec)
for i in range(num_times):
these_current_rows = numpy.where(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values ==
valid_times_unix_sec[i])[0]
these_current_subrows = _filter_storm_objects_latlng(
storm_object_table=storm_object_table.iloc[these_current_rows],
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg)
if len(these_current_subrows) == 0:
continue
these_current_rows = these_current_rows[these_current_subrows]
this_storm_object_table = _find_relevant_storm_objects(
storm_object_table=storm_object_table,
current_rows=these_current_rows)
these_latlng_rows = _filter_storm_objects_latlng(
storm_object_table=this_storm_object_table,
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg)
if top_myrorss_dir_name is None:
this_radar_matrix = None
these_radar_latitudes_deg = None
these_radar_longitudes_deg = None
else:
this_myrorss_file_name = myrorss_and_mrms_io.find_raw_file(
top_directory_name=top_myrorss_dir_name,
unix_time_sec=valid_times_unix_sec[i],
spc_date_string=time_conversion.time_to_spc_date_string(
valid_times_unix_sec[i]),
field_name=radar_field_name,
data_source=radar_utils.MYRORSS_SOURCE_ID,
height_m_asl=radar_height_m_asl,
raise_error_if_missing=True)
print(
'Reading data from: "{0:s}"...'.format(this_myrorss_file_name))
this_metadata_dict = (
myrorss_and_mrms_io.read_metadata_from_raw_file(
netcdf_file_name=this_myrorss_file_name,
data_source=radar_utils.MYRORSS_SOURCE_ID))
this_sparse_grid_table = (
myrorss_and_mrms_io.read_data_from_sparse_grid_file(
netcdf_file_name=this_myrorss_file_name,
field_name_orig=this_metadata_dict[
myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG],
data_source=radar_utils.MYRORSS_SOURCE_ID,
sentinel_values=this_metadata_dict[
radar_utils.SENTINEL_VALUE_COLUMN]))
(this_radar_matrix, these_radar_latitudes_deg,
these_radar_longitudes_deg) = radar_s2f.sparse_to_full_grid(
sparse_grid_table=this_sparse_grid_table,
metadata_dict=this_metadata_dict)
this_radar_matrix = numpy.flipud(this_radar_matrix)
these_radar_latitudes_deg = these_radar_latitudes_deg[::-1]
_, this_axes_object, this_basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg,
resolution_string='i'))
_plot_storm_outlines_one_time(
storm_object_table=this_storm_object_table.iloc[these_latlng_rows],
valid_time_unix_sec=valid_times_unix_sec[i],
axes_object=this_axes_object,
basemap_object=this_basemap_object,
storm_colour=storm_colour,
storm_opacity=storm_opacity,
include_secondary_ids=include_secondary_ids,
output_dir_name=output_dir_name,
radar_matrix=this_radar_matrix,
radar_field_name=radar_field_name,
radar_latitudes_deg=these_radar_latitudes_deg,
radar_longitudes_deg=these_radar_longitudes_deg)
def _run(top_radar_dir_name, top_radar_dir_name_tarred,
top_echo_classifn_dir_name, first_spc_date_string,
last_spc_date_string, echo_top_field_name, min_echo_top_km,
min_size_pixels, min_intermax_distance_metres, max_velocity_diff_m_s01,
max_link_distance_m_s01, top_output_dir_name):
"""Tracks storms based on echo top.
This is effectively the main method.
:param top_radar_dir_name: See documentation at top of file.
:param top_radar_dir_name_tarred: Same.
:param top_echo_classifn_dir_name: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param echo_top_field_name: Same.
:param min_echo_top_km: Same.
:param min_size_pixels: Same.
:param min_intermax_distance_metres: Same.
:param max_velocity_diff_m_s01: Same.
:param max_link_distance_m_s01: Same.
:param top_output_dir_name: Same.
"""
if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES:
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
for this_spc_date_string in spc_date_strings:
this_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format(
top_radar_dir_name_tarred, this_spc_date_string[:4],
this_spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=this_tar_file_name,
field_names=[echo_top_field_name],
spc_date_string=this_spc_date_string,
top_target_directory_name=top_radar_dir_name)
print(SEPARATOR_STRING)
if top_echo_classifn_dir_name in ['', 'None']:
top_echo_classifn_dir_name = None
echo_top_tracking.run_tracking(
top_radar_dir_name=top_radar_dir_name,
top_output_dir_name=top_output_dir_name,
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string,
echo_top_field_name=echo_top_field_name,
top_echo_classifn_dir_name=top_echo_classifn_dir_name,
min_echo_top_km=min_echo_top_km,
min_intermax_distance_metres=min_intermax_distance_metres,
min_polygon_size_pixels=min_size_pixels,
max_velocity_diff_m_s01=max_velocity_diff_m_s01,
max_link_distance_m_s01=max_link_distance_m_s01,
min_track_duration_seconds=0)
print(SEPARATOR_STRING)
if echo_top_field_name in NATIVE_ECHO_TOP_FIELD_NAMES:
for this_spc_date_string in spc_date_strings:
myrorss_io.remove_unzipped_data_1day(
spc_date_string=this_spc_date_string,
top_directory_name=top_radar_dir_name,
field_names=[echo_top_field_name]
)
def _run(top_input_dir_name, tracking_scale_metres2, first_spc_date_string,
last_spc_date_string, min_distances_metres, max_distances_metres,
top_output_dir_name):
"""Creates one or more distance buffers around each storm object (polygon).
This is effectively the main method.
:param top_input_dir_name: See documentation at top of file.
:param tracking_scale_metres2: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param min_distances_metres: Same.
:param max_distances_metres: Same.
:param top_output_dir_name: Same.
"""
min_distances_metres[min_distances_metres < 0] = numpy.nan
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
for this_spc_date_string in spc_date_strings:
these_input_file_names = tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_input_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
if len(these_input_file_names) == 0:
continue
for this_input_file_name in these_input_file_names:
print('Reading input tracks from: "{0:s}"...'.format(
this_input_file_name))
this_storm_object_table = tracking_io.read_file(
this_input_file_name)
this_storm_object_table = (tracking_utils.create_distance_buffers(
storm_object_table=this_storm_object_table,
min_distances_metres=min_distances_metres,
max_distances_metres=max_distances_metres))
this_output_file_name = tracking_io.find_file(
top_tracking_dir_name=top_output_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
source_name=tracking_utils.SEGMOTION_NAME,
valid_time_unix_sec=tracking_io.file_name_to_time(
this_input_file_name),
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)
print('Writing input tracks + buffers to: "{0:s}"...\n'.format(
this_output_file_name))
tracking_io.write_file(storm_object_table=this_storm_object_table,
pickle_file_name=this_output_file_name)
print(SEPARATOR_STRING)
def _run(top_input_dir_name, target_name_for_downsampling,
first_spc_date_string, last_spc_date_string, downsampling_classes,
downsampling_fractions, for_training, top_output_dir_name):
"""Downsamples storm objects, based on target values.
This is effectively the main method.
:param top_input_dir_name: See documentation at top of file.
:param target_name_for_downsampling: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param downsampling_classes: Same.
:param downsampling_fractions: Same.
:param for_training: Same.
:param top_output_dir_name: Same.
"""
all_spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
downsampling_dict = dict(
list(zip(downsampling_classes, downsampling_fractions)))
target_param_dict = target_val_utils.target_name_to_params(
target_name_for_downsampling)
event_type_string = target_param_dict[target_val_utils.EVENT_TYPE_KEY]
input_target_file_names = []
spc_date_string_by_file = []
for this_spc_date_string in all_spc_date_strings:
this_file_name = target_val_utils.find_target_file(
top_directory_name=top_input_dir_name,
event_type_string=event_type_string,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)
if not os.path.isfile(this_file_name):
continue
input_target_file_names.append(this_file_name)
spc_date_string_by_file.append(this_spc_date_string)
num_files = len(input_target_file_names)
target_dict_by_file = [None] * num_files
full_id_strings = []
storm_times_unix_sec = numpy.array([], dtype=int)
storm_to_file_indices = numpy.array([], dtype=int)
target_names = []
target_matrix = None
for i in range(num_files):
print('Reading data from: "{0:s}"...'.format(
input_target_file_names[i]))
target_dict_by_file[i] = target_val_utils.read_target_values(
netcdf_file_name=input_target_file_names[i])
if i == 0:
target_names = (
target_dict_by_file[i][target_val_utils.TARGET_NAMES_KEY])
these_full_id_strings = (
target_dict_by_file[i][target_val_utils.FULL_IDS_KEY])
full_id_strings += these_full_id_strings
this_num_storm_objects = len(these_full_id_strings)
storm_times_unix_sec = numpy.concatenate(
(storm_times_unix_sec,
target_dict_by_file[i][target_val_utils.VALID_TIMES_KEY]))
storm_to_file_indices = numpy.concatenate(
(storm_to_file_indices,
numpy.full(this_num_storm_objects, i, dtype=int)))
this_target_matrix = (
target_dict_by_file[i][target_val_utils.TARGET_MATRIX_KEY])
if target_matrix is None:
target_matrix = this_target_matrix + 0
else:
target_matrix = numpy.concatenate(
(target_matrix, this_target_matrix), axis=0)
print(SEPARATOR_STRING)
downsampling_index = target_names.index(target_name_for_downsampling)
good_indices = numpy.where(target_matrix[:, downsampling_index] !=
target_val_utils.INVALID_STORM_INTEGER)[0]
full_id_strings = [full_id_strings[k] for k in good_indices]
storm_times_unix_sec = storm_times_unix_sec[good_indices]
target_matrix = target_matrix[good_indices, :]
storm_to_file_indices = storm_to_file_indices[good_indices]
primary_id_strings = temporal_tracking.full_to_partial_ids(
full_id_strings)[0]
if for_training:
indices_to_keep = fancy_downsampling.downsample_for_training(
primary_id_strings=primary_id_strings,
storm_times_unix_sec=storm_times_unix_sec,
target_values=target_matrix[:, downsampling_index],
target_name=target_name_for_downsampling,
class_fraction_dict=downsampling_dict)
else:
indices_to_keep = fancy_downsampling.downsample_for_non_training(
primary_id_strings=primary_id_strings,
storm_times_unix_sec=storm_times_unix_sec,
target_values=target_matrix[:, downsampling_index],
target_name=target_name_for_downsampling,
class_fraction_dict=downsampling_dict)
print(SEPARATOR_STRING)
for i in range(num_files):
these_object_subindices = numpy.where(
storm_to_file_indices[indices_to_keep] == i)[0]
these_object_indices = indices_to_keep[these_object_subindices]
if len(these_object_indices) == 0:
continue
these_indices_in_file = tracking_utils.find_storm_objects(
all_id_strings=target_dict_by_file[i][
target_val_utils.FULL_IDS_KEY],
all_times_unix_sec=target_dict_by_file[i][
target_val_utils.VALID_TIMES_KEY],
id_strings_to_keep=[
full_id_strings[k] for k in these_object_indices
],
times_to_keep_unix_sec=storm_times_unix_sec[these_object_indices],
allow_missing=False)
this_output_dict = {
tracking_utils.FULL_ID_COLUMN: [
target_dict_by_file[i][target_val_utils.FULL_IDS_KEY][k]
for k in these_indices_in_file
],
tracking_utils.VALID_TIME_COLUMN:
target_dict_by_file[i][target_val_utils.VALID_TIMES_KEY]
[these_indices_in_file]
}
for j in range(len(target_names)):
this_output_dict[target_names[j]] = (target_dict_by_file[i][
target_val_utils.TARGET_MATRIX_KEY][these_indices_in_file, j])
this_output_table = pandas.DataFrame.from_dict(this_output_dict)
this_new_file_name = target_val_utils.find_target_file(
top_directory_name=top_output_dir_name,
event_type_string=event_type_string,
spc_date_string=spc_date_string_by_file[i],
raise_error_if_missing=False)
print((
'Writing {0:d} downsampled storm objects (out of {1:d} total) to: '
'"{2:s}"...').format(
len(this_output_table.index),
len(target_dict_by_file[i][target_val_utils.FULL_IDS_KEY]),
this_new_file_name))
target_val_utils.write_target_values(
storm_to_events_table=this_output_table,
target_names=target_names,
netcdf_file_name=this_new_file_name)
def _find_tracking_files_one_example(top_tracking_dir_name,
valid_time_unix_sec, target_name):
"""Finds tracking files needed to make plots for one example.
:param top_tracking_dir_name: See documentation at top of file.
:param valid_time_unix_sec: Valid time for example.
:param target_name: Name of target variable.
:return: tracking_file_names: 1-D list of paths to tracking files.
:raises: ValueError: if no tracking files are found.
"""
target_param_dict = target_val_utils.target_name_to_params(target_name)
min_lead_time_seconds = target_param_dict[
target_val_utils.MIN_LEAD_TIME_KEY]
max_lead_time_seconds = target_param_dict[
target_val_utils.MAX_LEAD_TIME_KEY]
first_time_unix_sec = valid_time_unix_sec + min_lead_time_seconds
last_time_unix_sec = valid_time_unix_sec + max_lead_time_seconds
first_spc_date_string = time_conversion.time_to_spc_date_string(
first_time_unix_sec - TIME_INTERVAL_SECONDS)
last_spc_date_string = time_conversion.time_to_spc_date_string(
last_time_unix_sec + TIME_INTERVAL_SECONDS)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
tracking_file_names = []
for this_spc_date_string in spc_date_strings:
these_file_names = tracking_io.find_files_one_spc_date(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=echo_top_tracking.
DUMMY_TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)[0]
tracking_file_names += these_file_names
if len(tracking_file_names) == 0:
error_string = (
'Cannot find any tracking files for SPC dates "{0:s}" to "{1:s}".'
).format(first_spc_date_string, last_spc_date_string)
raise ValueError(error_string)
tracking_times_unix_sec = numpy.array(
[tracking_io.file_name_to_time(f) for f in tracking_file_names],
dtype=int)
sort_indices = numpy.argsort(tracking_times_unix_sec)
tracking_times_unix_sec = tracking_times_unix_sec[sort_indices]
tracking_file_names = [tracking_file_names[k] for k in sort_indices]
these_indices = numpy.where(
tracking_times_unix_sec <= first_time_unix_sec)[0]
if len(these_indices) == 0:
first_index = 0
else:
first_index = these_indices[-1]
these_indices = numpy.where(
tracking_times_unix_sec >= last_time_unix_sec)[0]
if len(these_indices) == 0:
last_index = len(tracking_file_names) - 1
else:
last_index = these_indices[0]
return tracking_file_names[first_index:(last_index + 1)]
def _run(top_linkage_dir_name, first_spc_date_string, last_spc_date_string,
min_lead_times_sec, max_lead_times_sec, min_link_distances_metres,
max_link_distances_metres, event_type_string,
wind_speed_percentile_level, wind_speed_cutoffs_kt,
top_output_dir_name):
"""Computes target value for ea storm object, lead-time window, and buffer.
This is effectively the main method.
:param top_linkage_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param min_lead_times_sec: Same.
:param max_lead_times_sec: Same.
:param min_link_distances_metres: Same.
:param max_link_distances_metres: Same.
:param event_type_string: Same.
:param wind_speed_percentile_level: Same.
:param wind_speed_cutoffs_kt: Same.
:param top_output_dir_name: Same.
"""
num_lead_time_windows = len(min_lead_times_sec)
these_expected_dim = numpy.array([num_lead_time_windows], dtype=int)
error_checking.assert_is_numpy_array(max_lead_times_sec,
exact_dimensions=these_expected_dim)
num_distance_buffers = len(min_link_distances_metres)
these_expected_dim = numpy.array([num_distance_buffers], dtype=int)
error_checking.assert_is_numpy_array(max_link_distances_metres,
exact_dimensions=these_expected_dim)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
for this_spc_date_string in spc_date_strings:
this_linkage_file_name = linkage.find_linkage_file(
top_directory_name=top_linkage_dir_name,
event_type_string=event_type_string,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)
if not os.path.isfile(this_linkage_file_name):
continue
print('Reading data from: "{0:s}"...'.format(this_linkage_file_name))
this_storm_to_events_table = linkage.read_linkage_file(
this_linkage_file_name)[0]
_compute_targets_one_day(
storm_to_events_table=this_storm_to_events_table,
spc_date_string=this_spc_date_string,
min_lead_times_sec=min_lead_times_sec,
max_lead_times_sec=max_lead_times_sec,
min_link_distances_metres=min_link_distances_metres,
max_link_distances_metres=max_link_distances_metres,
event_type_string=event_type_string,
wind_speed_percentile_level=wind_speed_percentile_level,
wind_speed_cutoffs_kt=wind_speed_cutoffs_kt,
top_output_dir_name=top_output_dir_name)
FIRST_SPC_DATE_STRING = '20110601'
LAST_SPC_DATE_STRING = '20110831'
TOP_PROCESSED_DIR_NAME = '/condo/swatwork/ralager/myrorss_40dbz_echo_tops/final_tracks/reanalyzed/'
TRACKING_SCALE_METRES2 = 314159265
HOUR_FORMAT = '%H'
NUM_HOURS_PER_DAY = 24
BIRTH_CLIMATOLOGY_TYPE = 'birth'
DEATH_CLIMATOLOGY_TYPE = 'death'
PASSAGE_CLIMATOLOGY_TYPE = 'passage'
if __name__ == '__main__':
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=FIRST_SPC_DATE_STRING,
last_spc_date_string=LAST_SPC_DATE_STRING)
num_spc_dates = len(spc_date_strings)
storm_object_table_by_spc_date = [None] * num_spc_dates
num_storm_objects_by_hour = numpy.full(NUM_HOURS_PER_DAY, 0, dtype=int)
speeds = []
for working_date_index in range(num_spc_dates):
date_in_memory_indices = utils._get_dates_needed(
working_date_index=working_date_index,
num_dates=num_spc_dates,
climatology_type=PASSAGE_CLIMATOLOGY_TYPE)
for i in range(num_spc_dates):
if i in date_in_memory_indices:
def _find_tracking_gaps(first_spc_date_string, last_spc_date_string,
top_tracking_dir_name, tracking_scale_metres2,
source_name, min_time_diff_seconds):
"""Finds gaps (temporal discontinuities) between storm-tracking files.
:param first_spc_date_string: See documentation at top of file.
:param last_spc_date_string: Same.
:param top_tracking_dir_name: Same.
:param tracking_scale_metres2: Same.
:param source_name: Same.
:param min_time_diff_seconds: Same.
"""
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
tracking_file_names = []
unix_times_sec = numpy.array([], dtype=int)
num_spc_dates = len(spc_date_strings)
for i in range(num_spc_dates):
print('Finding tracking files for SPC date "{0:s}"...'.format(
spc_date_strings[i]))
these_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_strings[i],
source_name=source_name,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
raise_error_if_missing=False)[0]
print(len(these_file_names))
if not len(these_file_names):
continue
these_file_sizes_bytes = numpy.array(
[os.path.getsize(f) for f in these_file_names], dtype=int)
these_valid_indices = numpy.where(
these_file_sizes_bytes > FILE_SIZE_WITHOUT_STORMS_BYTES)[0]
these_file_names = [these_file_names[k] for k in these_valid_indices]
these_unix_times_sec = numpy.array(
[tracking_io.file_name_to_time(f) for f in these_file_names],
dtype=int)
these_sort_indices = numpy.argsort(these_unix_times_sec)
these_unix_times_sec = these_unix_times_sec[these_sort_indices]
these_file_names = [these_file_names[k] for k in these_sort_indices]
tracking_file_names += these_file_names
unix_times_sec = numpy.concatenate(
(unix_times_sec, these_unix_times_sec))
time_diffs_seconds = numpy.diff(unix_times_sec)
time_gap_indices = numpy.where(
time_diffs_seconds >= min_time_diff_seconds)[0]
num_time_gaps = len(time_gap_indices)
print((
'\nThere are {0:d} time gaps (successive files >= {1:d} seconds apart),'
' listed below:\n').format(num_time_gaps, min_time_diff_seconds))
for i in time_gap_indices:
this_start_time_string = time_conversion.unix_sec_to_string(
unix_times_sec[i], TIME_FORMAT)
this_end_time_string = time_conversion.unix_sec_to_string(
unix_times_sec[i + 1], TIME_FORMAT)
print('Gap between {0:s} and {1:s} = {2:d} seconds'.format(
this_start_time_string, this_end_time_string,
time_diffs_seconds[i]))
def _run(top_linkage_dir_name, genesis_only, max_link_distance_metres,
first_spc_date_string, last_spc_date_string, num_colours,
min_plot_latitude_deg, max_plot_latitude_deg, min_plot_longitude_deg,
max_plot_longitude_deg, output_file_name):
"""Plots tornado reports, storm tracks, and linkages.
This is effectively the main method.
:param top_linkage_dir_name: See documentation at top of file.
:param genesis_only: Same.
:param max_link_distance_metres: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param num_colours: Same.
:param min_plot_latitude_deg: Same.
:param max_plot_latitude_deg: Same.
:param min_plot_longitude_deg: Same.
:param max_plot_longitude_deg: Same.
:param output_file_name: Same.
"""
if max_link_distance_metres < 0:
max_link_distance_metres = None
colour_map_object = _truncate_colour_map(
orig_colour_map_object=pyplot.cm.get_cmap('YlOrRd'),
num_colours=num_colours)
event_type_string = (linkage.TORNADOGENESIS_EVENT_STRING
if genesis_only else linkage.TORNADO_EVENT_STRING)
if min_plot_latitude_deg <= SENTINEL_VALUE:
min_plot_latitude_deg = None
if max_plot_latitude_deg <= SENTINEL_VALUE:
max_plot_latitude_deg = None
if min_plot_longitude_deg <= SENTINEL_VALUE:
min_plot_longitude_deg = None
if max_plot_longitude_deg <= SENTINEL_VALUE:
max_plot_longitude_deg = None
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
list_of_linkage_tables = []
list_of_tornado_tables = []
linkage_metadata_dict = None
for this_spc_date_string in spc_date_strings:
this_file_name = linkage.find_linkage_file(
top_directory_name=top_linkage_dir_name,
event_type_string=event_type_string,
spc_date_string=this_spc_date_string,
raise_error_if_missing=False)
if not os.path.isfile(this_file_name):
continue
print('Reading data from: "{0:s}"...'.format(this_file_name))
this_linkage_table, linkage_metadata_dict, this_tornado_table = (
linkage.read_linkage_file(this_file_name))
list_of_linkage_tables.append(this_linkage_table)
list_of_tornado_tables.append(this_tornado_table)
if len(list_of_linkage_tables) == 1:
continue
list_of_linkage_tables[-1] = list_of_linkage_tables[-1].align(
list_of_linkage_tables[0], axis=1)[0]
list_of_tornado_tables[-1] = list_of_tornado_tables[-1].align(
list_of_tornado_tables[0], axis=1)[0]
print(SEPARATOR_STRING)
storm_to_tornadoes_table = pandas.concat(list_of_linkage_tables,
axis=0,
ignore_index=True)
tornado_table = pandas.concat(list_of_tornado_tables,
axis=0,
ignore_index=True)
column_dict_old_to_new = {
linkage.EVENT_TIME_COLUMN: tornado_io.TIME_COLUMN,
linkage.EVENT_LATITUDE_COLUMN: tornado_io.LATITUDE_COLUMN,
linkage.EVENT_LONGITUDE_COLUMN: tornado_io.LONGITUDE_COLUMN
}
tornado_table.rename(columns=column_dict_old_to_new, inplace=True)
tornado_table = tornado_io.segments_to_tornadoes(tornado_table)
tornado_table = tornado_table.assign(
**{
SHORT_TORNADO_ID_COLUMN:
_long_to_short_tornado_ids(tornado_table[
tornado_io.TORNADO_ID_COLUMN].values)
})
if min_plot_latitude_deg is None:
min_plot_latitude_deg = numpy.min(
storm_to_tornadoes_table[tracking_utils.CENTROID_LATITUDE_COLUMN].
values) - LATLNG_BUFFER_DEG
if max_plot_latitude_deg is None:
max_plot_latitude_deg = numpy.max(
storm_to_tornadoes_table[tracking_utils.CENTROID_LATITUDE_COLUMN].
values) + LATLNG_BUFFER_DEG
if min_plot_longitude_deg is None:
min_plot_longitude_deg = numpy.min(
storm_to_tornadoes_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].
values) - LATLNG_BUFFER_DEG
if max_plot_longitude_deg is None:
max_plot_longitude_deg = numpy.max(
storm_to_tornadoes_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].
values) + LATLNG_BUFFER_DEG
# TODO(thunderhoser): Should maybe restrict this to an inner domain.
storm_to_tornadoes_table = storm_to_tornadoes_table.loc[
(storm_to_tornadoes_table[tracking_utils.CENTROID_LATITUDE_COLUMN] >=
min_plot_latitude_deg)
& (storm_to_tornadoes_table[tracking_utils.CENTROID_LATITUDE_COLUMN] <=
max_plot_latitude_deg)]
storm_to_tornadoes_table = storm_to_tornadoes_table.loc[
(storm_to_tornadoes_table[tracking_utils.CENTROID_LONGITUDE_COLUMN] >=
min_plot_longitude_deg)
& (storm_to_tornadoes_table[tracking_utils.CENTROID_LONGITUDE_COLUMN]
<= max_plot_longitude_deg)]
tornado_io.subset_tornadoes(tornado_table=tornado_table,
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg)
# TODO(thunderhoser): Make this subsetting optional.
storm_to_tornadoes_table = _subset_storms_by_time(
storm_to_tornadoes_table=storm_to_tornadoes_table,
tornado_table=tornado_table,
linkage_metadata_dict=linkage_metadata_dict,
genesis_only=genesis_only)
print(SEPARATOR_STRING)
_, axes_object, basemap_object = (
plotting_utils.create_equidist_cylindrical_map(
min_latitude_deg=min_plot_latitude_deg,
max_latitude_deg=max_plot_latitude_deg,
min_longitude_deg=min_plot_longitude_deg,
max_longitude_deg=max_plot_longitude_deg,
resolution_string='i'))
plotting_utils.plot_coastlines(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_countries(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_states_and_provinces(basemap_object=basemap_object,
axes_object=axes_object,
line_colour=BORDER_COLOUR)
plotting_utils.plot_parallels(basemap_object=basemap_object,
axes_object=axes_object,
num_parallels=NUM_PARALLELS)
plotting_utils.plot_meridians(basemap_object=basemap_object,
axes_object=axes_object,
num_meridians=NUM_MERIDIANS)
print('Plotting storm tracks...')
storm_plotting.plot_storm_tracks(
storm_object_table=storm_to_tornadoes_table,
axes_object=axes_object,
basemap_object=basemap_object,
colour_map_object=colour_map_object,
start_marker_type=None,
end_marker_type=None)
num_tornadoes = len(tornado_table.index)
if num_tornadoes == 0:
print('Saving figure to: "{0:s}"...'.format(output_file_name))
pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
return
colour_norm_object = pyplot.Normalize(
numpy.min(
storm_to_tornadoes_table[tracking_utils.VALID_TIME_COLUMN].values),
numpy.max(
storm_to_tornadoes_table[tracking_utils.VALID_TIME_COLUMN].values))
print('Plotting tornado markers...')
_plot_tornadoes(tornado_table=tornado_table,
colour_map_object=colour_map_object,
colour_norm_object=colour_norm_object,
genesis_only=genesis_only,
axes_object=axes_object,
basemap_object=basemap_object)
print('Plotting tornado IDs with storm objects...')
num_storm_objects = len(storm_to_tornadoes_table.index)
for i in range(num_storm_objects):
_plot_linkages_one_storm_object(
storm_to_tornadoes_table=storm_to_tornadoes_table,
storm_object_index=i,
tornado_table=tornado_table,
colour_map_object=colour_map_object,
colour_norm_object=colour_norm_object,
axes_object=axes_object,
basemap_object=basemap_object,
max_link_distance_metres=max_link_distance_metres)
print('Saving figure to: "{0:s}"...'.format(output_file_name))
pyplot.savefig(output_file_name, dpi=FIGURE_RESOLUTION_DPI)
pyplot.close()
def _run(myrorss_tracking_dir_name, gridrad_tracking_dir_name,
max_distance_metres, source_dataset_name, first_spc_date_string,
last_spc_date_string, output_dir_name):
"""Matches storm objects between MYRORSS and GridRad datasets.
This is effectively the main method.
:param myrorss_tracking_dir_name: See documentation at end of file.
:param gridrad_tracking_dir_name: Same.
:param max_distance_metres: Same.
:param source_dataset_name: Same.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param output_dir_name: Same.
:raises: ValueError: if `source_dataset_name not in VALID_DATASET_NAMES`.
"""
if source_dataset_name not in VALID_DATASET_NAMES:
error_string = (
'\n{0:s}\nValid datasets (listed above) do not include "{1:s}".'
).format(str(VALID_DATASET_NAMES), source_dataset_name)
raise ValueError(error_string)
spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
if source_dataset_name == radar_utils.MYRORSS_SOURCE_ID:
source_tracking_dir_name = myrorss_tracking_dir_name
target_tracking_dir_name = gridrad_tracking_dir_name
target_dataset_name = radar_utils.GRIDRAD_SOURCE_ID
else:
source_tracking_dir_name = gridrad_tracking_dir_name
target_tracking_dir_name = myrorss_tracking_dir_name
target_dataset_name = radar_utils.MYRORSS_SOURCE_ID
source_tracking_file_names = []
target_tracking_file_names = []
for this_spc_date_string in spc_date_strings:
source_tracking_file_names += tracking_io.find_files_one_spc_date(
top_tracking_dir_name=source_tracking_dir_name,
tracking_scale_metres2=TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=True)[0]
target_tracking_file_names += tracking_io.find_files_one_spc_date(
top_tracking_dir_name=target_tracking_dir_name,
tracking_scale_metres2=TRACKING_SCALE_METRES2,
source_name=tracking_utils.SEGMOTION_NAME,
spc_date_string=this_spc_date_string,
raise_error_if_missing=True)[0]
source_times_unix_sec = numpy.array(
[tracking_io.file_name_to_time(f) for f in source_tracking_file_names],
dtype=int)
target_times_unix_sec = numpy.array(
[tracking_io.file_name_to_time(f) for f in target_tracking_file_names],
dtype=int)
source_to_target_indices = _match_all_times(
source_times_unix_sec=source_times_unix_sec,
target_times_unix_sec=target_times_unix_sec,
max_diff_seconds=MAX_TIME_DIFF_SECONDS)
print(SEPARATOR_STRING)
del target_times_unix_sec
target_tracking_file_names = [
target_tracking_file_names[k] for k in source_to_target_indices
]
num_source_times = len(source_times_unix_sec)
for i in range(num_source_times):
print('Reading data from: "{0:s}"...'.format(
source_tracking_file_names[i]))
this_source_object_table = tracking_io.read_file(
source_tracking_file_names[i])
print('Reading data from: "{0:s}"...'.format(
target_tracking_file_names[i]))
this_target_object_table = tracking_io.read_file(
target_tracking_file_names[i])
this_source_to_target_dict = _match_locations_one_time(
source_object_table=this_source_object_table,
target_object_table=this_target_object_table,
max_distance_metres=max_distance_metres)
this_match_file_name = tracking_io.find_match_file(
top_directory_name=output_dir_name,
valid_time_unix_sec=source_times_unix_sec[i],
raise_error_if_missing=False)
print('Writing results to: "{0:s}"...\n'.format(this_match_file_name))
tracking_io.write_matches(
pickle_file_name=this_match_file_name,
source_to_target_dict=this_source_to_target_dict,
max_time_diff_seconds=MAX_TIME_DIFF_SECONDS,
max_distance_metres=max_distance_metres,
source_dataset_name=source_dataset_name,
source_tracking_dir_name=source_tracking_dir_name,
target_dataset_name=target_dataset_name,
target_tracking_dir_name=target_tracking_dir_name)
