def _find_gridrad_file_for_date(top_gridrad_dir_name, spc_date_string):
"""Tries to find one GridRad file for given SPC date.
:param top_gridrad_dir_name: See documentation at top of file.
:param spc_date_string: SPC date or convective day (format "yyyymmdd").
:return: gridrad_file_name: Path to GridRad file. If no files were found
for the given SPC date, returns None.
"""
first_time_unix_sec = time_conversion.get_start_of_spc_date(
spc_date_string)
last_time_unix_sec = time_conversion.get_end_of_spc_date(spc_date_string)
all_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_time_unix_sec,
end_time_unix_sec=last_time_unix_sec,
time_interval_sec=TIME_INTERVAL_SEC,
include_endpoint=True)
for this_time_unix_sec in all_times_unix_sec:
this_gridrad_file_name = gridrad_io.find_file(
unix_time_sec=this_time_unix_sec,
top_directory_name=top_gridrad_dir_name,
raise_error_if_missing=False)
if os.path.isfile(this_gridrad_file_name):
return this_gridrad_file_name
return None
def test_find_file(self):
"""Ensures correct output from find_file."""
this_file_name = gridrad_io.find_file(
unix_time_sec=UNIX_TIME_SEC,
top_directory_name=TOP_DIRECTORY_NAME,
raise_error_if_missing=False)
self.assertTrue(this_file_name == FULL_FILE_NAME)
def get_storm_based_radar_stats_gridrad(
storm_object_table,
top_radar_dir_name,
statistic_names=DEFAULT_STATISTIC_NAMES,
percentile_levels=DEFAULT_PERCENTILE_LEVELS,
radar_field_names=DEFAULT_FIELDS_FOR_GRIDRAD,
radar_heights_m_asl=DEFAULT_HEIGHTS_FOR_GRIDRAD_M_ASL):
"""Computes radar statistics for each storm object.
In this case, radar data must be from GridRad.
N = number of storm objects
F = number of radar fields
H = number of radar heights
S = number of statistics (percentile- and non-percentile-based)
:param storm_object_table: N-row pandas DataFrame with columns listed in
`storm_tracking_io.write_file`. Each row is one storm object.
:param top_radar_dir_name: [input] Name of top-level directory with radar
data from the given source.
:param statistic_names: 1-D list of non-percentile-based statistics.
:param percentile_levels: 1-D numpy array of percentile levels.
:param radar_field_names: length-F list of radar fields for which stats will
be computed.
:param radar_heights_m_asl: length-H numpy array of radar heights (metres
above sea level).
:return: storm_object_statistic_table: pandas DataFrame with 2 + S * F * H
columns. The last S * F * H columns are one for each statistic-field-
height tuple. Names of these columns are determined by
`radar_field_and_statistic_to_column_name` and
`radar_field_and_percentile_to_column_name`. The first 2 columns are
listed below.
storm_object_statistic_table.full_id_string: Storm ID (taken from input
table).
storm_object_statistic_table.unix_time_sec: Valid time (taken from input
table).
"""
# Error-checking.
percentile_levels = _check_statistic_params(statistic_names,
percentile_levels)
_, _ = gridrad_utils.fields_and_refl_heights_to_pairs(
field_names=radar_field_names, heights_m_asl=radar_heights_m_asl)
radar_heights_m_asl = numpy.sort(
numpy.round(radar_heights_m_asl).astype(int))
# Find radar files.
radar_times_unix_sec = numpy.unique(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values)
radar_time_strings = [
time_conversion.unix_sec_to_string(t, DEFAULT_TIME_FORMAT)
for t in radar_times_unix_sec
]
num_radar_times = len(radar_times_unix_sec)
radar_file_names = [None] * num_radar_times
for i in range(num_radar_times):
radar_file_names[i] = gridrad_io.find_file(
unix_time_sec=radar_times_unix_sec[i],
top_directory_name=top_radar_dir_name,
raise_error_if_missing=True)
# Initialize output.
num_radar_fields = len(radar_field_names)
num_radar_heights = len(radar_heights_m_asl)
num_statistics = len(statistic_names)
num_percentiles = len(percentile_levels)
num_storm_objects = len(storm_object_table.index)
statistic_matrix = numpy.full((num_storm_objects, num_radar_fields,
num_radar_heights, num_statistics),
numpy.nan)
percentile_matrix = numpy.full((num_storm_objects, num_radar_fields,
num_radar_heights, num_percentiles),
numpy.nan)
for i in range(num_radar_times):
# Read metadata for [i]th valid time and find storm objects at [i]th
# valid time.
this_metadata_dict = gridrad_io.read_metadata_from_full_grid_file(
radar_file_names[i])
these_storm_indices = numpy.where(
storm_object_table[tracking_utils.VALID_TIME_COLUMN].values ==
radar_times_unix_sec[i])[0]
for j in range(num_radar_fields):
# Read data for [j]th field at [i]th valid time.
print('Reading "{0:s}" from file "{1:s}"...'.format(
radar_field_names[j], radar_time_strings[i]))
radar_matrix_this_field, these_grid_point_heights_m_asl, _, _ = (
gridrad_io.read_field_from_full_grid_file(
radar_file_names[i],
field_name=radar_field_names[j],
metadata_dict=this_metadata_dict))
these_grid_point_heights_m_asl = numpy.round(
these_grid_point_heights_m_asl).astype(int)
these_height_indices_to_keep = numpy.array([
these_grid_point_heights_m_asl.tolist().index(h)
for h in radar_heights_m_asl
],
dtype=int)
del these_grid_point_heights_m_asl
radar_matrix_this_field = (
radar_matrix_this_field[these_height_indices_to_keep, :, :])
radar_matrix_this_field[numpy.isnan(radar_matrix_this_field)] = 0.
for k in range(num_radar_heights):
# Compute radar stats for [j]th field at [k]th height and [i]th
# valid time.
print(('Computing stats for "{0:s}" at {1:d} metres ASL and '
'{2:s}...').format(radar_field_names[j],
radar_heights_m_asl[k],
radar_time_strings[i]))
for this_storm_index in these_storm_indices:
these_grid_point_rows = storm_object_table[
tracking_utils.ROWS_IN_STORM_COLUMN].values[
this_storm_index].astype(int)
these_grid_point_columns = storm_object_table[
tracking_utils.COLUMNS_IN_STORM_COLUMN].values[
this_storm_index].astype(int)
radar_values_this_storm = extract_radar_grid_points(
field_matrix=numpy.flipud(
radar_matrix_this_field[k, :, :]),
row_indices=these_grid_point_rows,
column_indices=these_grid_point_columns)
(statistic_matrix[this_storm_index, j, k, :],
percentile_matrix[this_storm_index, j,
k, :]) = get_spatial_statistics(
radar_values_this_storm,
statistic_names=statistic_names,
percentile_levels=percentile_levels)
print('\n')
# Create pandas DataFrame.
storm_object_statistic_dict = {}
for j in range(num_radar_fields):
for k in range(num_radar_heights):
for m in range(num_statistics):
this_column_name = radar_field_and_statistic_to_column_name(
radar_field_name=radar_field_names[j],
radar_height_m_asl=radar_heights_m_asl[k],
statistic_name=statistic_names[m])
storm_object_statistic_dict.update(
{this_column_name: statistic_matrix[:, j, k, m]})
for m in range(num_percentiles):
this_column_name = radar_field_and_percentile_to_column_name(
radar_field_name=radar_field_names[j],
radar_height_m_asl=radar_heights_m_asl[k],
percentile_level=percentile_levels[m])
storm_object_statistic_dict.update(
{this_column_name: percentile_matrix[:, j, k, m]})
storm_object_statistic_table = pandas.DataFrame.from_dict(
storm_object_statistic_dict)
return pandas.concat([
storm_object_table[STORM_COLUMNS_TO_KEEP], storm_object_statistic_table
],
axis=1)
def _run_for_gridrad(spc_date_string, top_radar_dir_name, top_output_dir_name,
option_dict):
"""Runs echo classification for GridRad data.
:param spc_date_string: See documentation at top of file.
:param top_radar_dir_name: Same.
:param top_output_dir_name: Same.
:param option_dict: See doc for
`echo_classification.find_convective_pixels`.
"""
valid_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=time_conversion.get_start_of_spc_date(
spc_date_string),
end_time_unix_sec=time_conversion.get_end_of_spc_date(spc_date_string),
time_interval_sec=TIME_INTERVAL_SEC,
include_endpoint=True)
num_times = len(valid_times_unix_sec)
radar_file_names = [''] * num_times
indices_to_keep = []
for i in range(num_times):
radar_file_names[i] = gridrad_io.find_file(
top_directory_name=top_radar_dir_name,
unix_time_sec=valid_times_unix_sec[i],
raise_error_if_missing=False)
if os.path.isfile(radar_file_names[i]):
indices_to_keep.append(i)
indices_to_keep = numpy.array(indices_to_keep, dtype=int)
valid_times_unix_sec = valid_times_unix_sec[indices_to_keep]
radar_file_names = [radar_file_names[k] for k in indices_to_keep]
num_times = len(valid_times_unix_sec)
for i in range(num_times):
print 'Reading data from: "{0:s}"...\n'.format(radar_file_names[i])
radar_metadata_dict = gridrad_io.read_metadata_from_full_grid_file(
netcdf_file_name=radar_file_names[i])
(reflectivity_matrix_dbz, all_heights_m_asl, grid_point_latitudes_deg,
grid_point_longitudes_deg
) = gridrad_io.read_field_from_full_grid_file(
netcdf_file_name=radar_file_names[i],
field_name=radar_utils.REFL_NAME,
metadata_dict=radar_metadata_dict)
reflectivity_matrix_dbz = numpy.rollaxis(reflectivity_matrix_dbz,
axis=0,
start=3)
height_indices = numpy.array(
[all_heights_m_asl.tolist().index(h) for h in RADAR_HEIGHTS_M_ASL],
dtype=int)
reflectivity_matrix_dbz = reflectivity_matrix_dbz[..., height_indices]
grid_metadata_dict = {
echo_classifn.MIN_LATITUDE_KEY:
numpy.min(grid_point_latitudes_deg),
echo_classifn.LATITUDE_SPACING_KEY:
grid_point_latitudes_deg[1] - grid_point_latitudes_deg[0],
echo_classifn.MIN_LONGITUDE_KEY:
numpy.min(grid_point_longitudes_deg),
echo_classifn.LONGITUDE_SPACING_KEY:
grid_point_longitudes_deg[1] - grid_point_longitudes_deg[0],
echo_classifn.HEIGHTS_KEY:
RADAR_HEIGHTS_M_ASL
}
convective_flag_matrix = echo_classifn.find_convective_pixels(
reflectivity_matrix_dbz=reflectivity_matrix_dbz,
grid_metadata_dict=grid_metadata_dict,
valid_time_unix_sec=valid_times_unix_sec[i],
option_dict=option_dict)
print 'Number of convective pixels = {0:d}\n'.format(
numpy.sum(convective_flag_matrix))
this_output_file_name = echo_classifn.find_classification_file(
top_directory_name=top_output_dir_name,
valid_time_unix_sec=valid_times_unix_sec[i],
desire_zipped=False,
allow_zipped_or_unzipped=False,
raise_error_if_missing=False)
print 'Writing echo classifications to: "{0:s}"...'.format(
this_output_file_name)
echo_classifn.write_classifications(
convective_flag_matrix=convective_flag_matrix,
grid_metadata_dict=grid_metadata_dict,
valid_time_unix_sec=valid_times_unix_sec[i],
option_dict=option_dict,
netcdf_file_name=this_output_file_name)
print SEPARATOR_STRING
def _run(top_gridrad_dir_name, first_spc_date_string, last_spc_date_string,
colour_map_name, grid_spacing_metres, output_file_name):
"""Plots GridRad domains.
This is effectively the main method.
:param top_gridrad_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
:param colour_map_name: Same.
:param grid_spacing_metres: Same.
:param output_file_name: Same.
"""
colour_map_object = pyplot.get_cmap(colour_map_name)
file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
first_time_unix_sec = time_conversion.get_start_of_spc_date(
first_spc_date_string)
last_time_unix_sec = time_conversion.get_end_of_spc_date(
last_spc_date_string)
valid_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_time_unix_sec,
end_time_unix_sec=last_time_unix_sec,
time_interval_sec=TIME_INTERVAL_SEC,
include_endpoint=True)
valid_spc_date_strings = [
time_conversion.time_to_spc_date_string(t)
for t in valid_times_unix_sec
]
domain_min_latitudes_deg = []
domain_max_latitudes_deg = []
domain_min_longitudes_deg = []
domain_max_longitudes_deg = []
prev_domain_limits_deg = numpy.full(4, numpy.nan)
prev_spc_date_string = 'foo'
num_times = len(valid_times_unix_sec)
for i in range(num_times):
this_gridrad_file_name = gridrad_io.find_file(
unix_time_sec=valid_times_unix_sec[i],
top_directory_name=top_gridrad_dir_name,
raise_error_if_missing=False)
if not os.path.isfile(this_gridrad_file_name):
continue
these_domain_limits_deg = _get_domain_one_file(this_gridrad_file_name)
same_domain = (valid_spc_date_strings[i] == prev_spc_date_string
and numpy.allclose(these_domain_limits_deg,
prev_domain_limits_deg, TOLERANCE))
if same_domain:
continue
prev_domain_limits_deg = these_domain_limits_deg + 0.
prev_spc_date_string = valid_spc_date_strings[i]
domain_min_latitudes_deg.append(these_domain_limits_deg[0])
domain_max_latitudes_deg.append(these_domain_limits_deg[1])
domain_min_longitudes_deg.append(these_domain_limits_deg[2])
domain_max_longitudes_deg.append(these_domain_limits_deg[3])
print(SEPARATOR_STRING)
domain_min_latitudes_deg = numpy.array(domain_min_latitudes_deg)
domain_max_latitudes_deg = numpy.array(domain_max_latitudes_deg)
domain_min_longitudes_deg = numpy.array(domain_min_longitudes_deg)
domain_max_longitudes_deg = numpy.array(domain_max_longitudes_deg)
num_domains = len(domain_min_latitudes_deg)
grid_metadata_dict = grids.create_equidistant_grid(
min_latitude_deg=OVERALL_MIN_LATITUDE_DEG,
max_latitude_deg=OVERALL_MAX_LATITUDE_DEG,
min_longitude_deg=OVERALL_MIN_LONGITUDE_DEG,
max_longitude_deg=OVERALL_MAX_LONGITUDE_DEG,
x_spacing_metres=grid_spacing_metres,
y_spacing_metres=grid_spacing_metres,
azimuthal=False)
unique_x_coords_metres = grid_metadata_dict[grids.X_COORDS_KEY]
unique_y_coords_metres = grid_metadata_dict[grids.Y_COORDS_KEY]
projection_object = grid_metadata_dict[grids.PROJECTION_KEY]
x_coord_matrix_metres, y_coord_matrix_metres = grids.xy_vectors_to_matrices(
x_unique_metres=unique_x_coords_metres,
y_unique_metres=unique_y_coords_metres)
latitude_matrix_deg, longitude_matrix_deg = (
projections.project_xy_to_latlng(x_coords_metres=x_coord_matrix_metres,
y_coords_metres=y_coord_matrix_metres,
projection_object=projection_object))
num_grid_rows = latitude_matrix_deg.shape[0]
num_grid_columns = latitude_matrix_deg.shape[1]
num_days_matrix = numpy.full((num_grid_rows, num_grid_columns), 0)
for i in range(num_domains):
if numpy.mod(i, 10) == 0:
print('Have found grid points in {0:d} of {1:d} domains...'.format(
i, num_domains))
this_lat_flag_matrix = numpy.logical_and(
latitude_matrix_deg >= domain_min_latitudes_deg[i],
latitude_matrix_deg <= domain_max_latitudes_deg[i])
this_lng_flag_matrix = numpy.logical_and(
longitude_matrix_deg >= domain_min_longitudes_deg[i],
longitude_matrix_deg <= domain_max_longitudes_deg[i])
num_days_matrix += numpy.logical_and(this_lat_flag_matrix,
this_lng_flag_matrix).astype(int)
print(SEPARATOR_STRING)
figure_object, axes_object = _plot_data(
num_days_matrix=num_days_matrix,
grid_metadata_dict=grid_metadata_dict,
colour_map_object=colour_map_object)
plotting_utils.label_axes(axes_object=axes_object, label_string='(c)')
print('Saving figure to: "{0:s}"...'.format(output_file_name))
figure_object.savefig(output_file_name,
dpi=FIGURE_RESOLUTION_DPI,
pad_inches=0,
bbox_inches='tight')
pyplot.close(figure_object)
def _find_domain_for_date(top_gridrad_dir_name, spc_date_string):
"""Finds GridRad domain for the given SPC date.
If no GridRad files are found the given the SPC date, this method returns
None for all output variables.
:param top_gridrad_dir_name: See documentation at top of file.
:param spc_date_string: SPC date or convective day (format "yyyymmdd").
:return: domain_limits_deg: length-4 numpy array with
[min latitude, max latitude, min longitude, max longitude].
Units are deg N for latitude, deg W for longitude.
"""
first_time_unix_sec = time_conversion.get_start_of_spc_date(spc_date_string)
last_time_unix_sec = time_conversion.get_end_of_spc_date(spc_date_string)
valid_times_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_time_unix_sec,
end_time_unix_sec=last_time_unix_sec,
time_interval_sec=TIME_INTERVAL_SEC, include_endpoint=True
)
num_times = len(valid_times_unix_sec)
min_latitudes_deg = numpy.full(num_times, numpy.nan)
max_latitudes_deg = numpy.full(num_times, numpy.nan)
min_longitudes_deg = numpy.full(num_times, numpy.nan)
max_longitudes_deg = numpy.full(num_times, numpy.nan)
for i in range(num_times):
this_file_name = gridrad_io.find_file(
unix_time_sec=valid_times_unix_sec[i],
top_directory_name=top_gridrad_dir_name,
raise_error_if_missing=False
)
if not os.path.isfile(this_file_name):
continue
print('Reading data from: "{0:s}"...'.format(this_file_name))
this_metadata_dict = gridrad_io.read_metadata_from_full_grid_file(
this_file_name
)
max_latitudes_deg[i] = (
this_metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN]
)
min_longitudes_deg[i] = (
this_metadata_dict[radar_utils.NW_GRID_POINT_LNG_COLUMN]
)
max_longitudes_deg[i] = min_longitudes_deg[i] + (
(this_metadata_dict[radar_utils.NUM_LNG_COLUMN] - 1) *
this_metadata_dict[radar_utils.LNG_SPACING_COLUMN]
)
min_latitudes_deg[i] = max_latitudes_deg[i] - (
(this_metadata_dict[radar_utils.NUM_LAT_COLUMN] - 1) *
this_metadata_dict[radar_utils.LAT_SPACING_COLUMN]
)
good_indices = numpy.where(numpy.invert(numpy.isnan(min_latitudes_deg)))[0]
if len(good_indices) == 0:
return None
coord_matrix = numpy.vstack((
min_latitudes_deg[good_indices], max_latitudes_deg[good_indices],
min_longitudes_deg[good_indices], max_longitudes_deg[good_indices]
))
coord_matrix, num_instances_by_row = numpy.unique(
numpy.transpose(coord_matrix), axis=0, return_counts=True
)
print(coord_matrix)
print(num_instances_by_row)
domain_limits_deg = coord_matrix[numpy.argmax(num_instances_by_row), :]
domain_limits_deg[2:] = -1 * lng_conversion.convert_lng_negative_in_west(
longitudes_deg=domain_limits_deg[2:], allow_nan=False
)
return domain_limits_deg