def test_get_valid_heights_storm_id(self):
"""Ensures correct output from get_valid_heights.
In this case, field is storm ID, which is not defined at certain
heights.
"""
with self.assertRaises(ValueError):
radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_utils.STORM_ID_NAME)
def test_get_valid_heights_gridrad(self):
"""Ensures correct output from get_valid_heights.
In this case, data source is GridRad.
"""
these_valid_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.GRIDRAD_SOURCE_ID)
self.assertTrue(len(these_valid_heights_m_asl) > 1)
def test_get_valid_heights_reflectivity(self):
"""Ensures correct output from get_valid_heights.
In this case, field is reflectivity (defined at many height levels).
"""
these_valid_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MRMS_SOURCE_ID,
field_name=radar_utils.REFL_NAME)
self.assertTrue(len(these_valid_heights_m_asl) > 1)
def test_get_valid_heights_mrms_shear(self):
"""Ensures correct output from get_valid_heights.
In this case, data source and field are azimuthal shear in MRMS.
"""
these_valid_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MRMS_SOURCE_ID,
field_name=radar_utils.MID_LEVEL_SHEAR_NAME)
self.assertTrue(
numpy.array_equal(these_valid_heights_m_asl, SHEAR_HEIGHTS_M_ASL))
def test_get_valid_heights_mrms_non_shear(self):
"""Ensures correct output from get_valid_heights.
In this case, data source is MRMS and field is *not* azimuthal shear.
"""
these_valid_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MRMS_SOURCE_ID,
field_name=radar_utils.REFL_M10CELSIUS_NAME)
self.assertTrue(numpy.array_equal(
these_valid_heights_m_asl, NON_SHEAR_HEIGHTS_MRMS_M_ASL))
def fields_and_refl_heights_to_pairs(field_names,
data_source,
refl_heights_m_asl=None):
"""Converts unique arrays (field names and refl heights) to non-unique ones.
F = number of fields
N = number of field-height pairs
:param field_names: length-F list with names of radar fields in
GewitterGefahr format.
:param data_source: Data source (string).
:param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres
above sea level).
:return: field_name_by_pair: length-N list of field names.
:return: height_by_pair_m_asl: length-N numpy array of corresponding heights
(metres above sea level).
"""
check_data_source(data_source)
error_checking.assert_is_string_list(field_names)
error_checking.assert_is_numpy_array(numpy.array(field_names),
num_dimensions=1)
field_name_by_pair = []
height_by_pair_m_asl = numpy.array([])
for this_field_name in field_names:
if this_field_name == radar_utils.REFL_NAME:
radar_utils.check_heights(data_source=data_source,
heights_m_asl=refl_heights_m_asl,
field_name=this_field_name)
these_heights_m_asl = copy.deepcopy(refl_heights_m_asl)
else:
these_heights_m_asl = radar_utils.get_valid_heights(
data_source=data_source, field_name=this_field_name)
field_name_by_pair += [this_field_name] * len(these_heights_m_asl)
height_by_pair_m_asl = numpy.concatenate(
(height_by_pair_m_asl, these_heights_m_asl))
return field_name_by_pair, height_by_pair_m_asl
def get_relative_dir_for_raw_files(field_name, data_source, height_m_asl=None):
"""Generates relative path for raw files.
:param field_name: Name of radar field in GewitterGefahr format.
:param data_source: Data source (string).
:param height_m_asl: Radar height (metres above sea level).
:return: relative_directory_name: Relative path for raw files.
"""
if field_name == radar_utils.REFL_NAME:
radar_utils.check_heights(data_source=data_source,
heights_m_asl=numpy.array([height_m_asl]),
field_name=radar_utils.REFL_NAME)
else:
height_m_asl = radar_utils.get_valid_heights(data_source=data_source,
field_name=field_name)[0]
return '{0:s}/{1:05.2f}'.format(
radar_utils.field_name_new_to_orig(field_name=field_name,
data_source_name=data_source),
float(height_m_asl) * METRES_TO_KM)
def fields_and_refl_heights_to_dict(field_names,
data_source,
refl_heights_m_asl=None):
"""Converts two arrays (field names and reflectivity heights) to dictionary.
:param field_names: 1-D list with names of radar fields in GewitterGefahr
format.
:param data_source: Data source (string).
:param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres
above sea level).
:return: field_to_heights_dict_m_asl: Dictionary, where each key is a field
name and each value is a 1-D numpy array of heights (metres above sea
level).
"""
check_data_source(data_source)
error_checking.assert_is_string_list(field_names)
error_checking.assert_is_numpy_array(numpy.array(field_names),
num_dimensions=1)
field_to_heights_dict_m_asl = {}
for this_field_name in field_names:
if this_field_name == radar_utils.REFL_NAME:
radar_utils.check_heights(data_source=data_source,
heights_m_asl=refl_heights_m_asl,
field_name=this_field_name)
field_to_heights_dict_m_asl.update(
{this_field_name: refl_heights_m_asl})
else:
field_to_heights_dict_m_asl.update({
this_field_name:
radar_utils.get_valid_heights(data_source=data_source,
field_name=this_field_name)
})
return field_to_heights_dict_m_asl
def get_echo_tops(
unix_time_sec,
spc_date_string,
top_directory_name,
critical_reflectivity_dbz,
top_height_to_consider_m_asl=DEFAULT_TOP_INPUT_HEIGHT_FOR_ECHO_TOPS_M_ASL,
lowest_refl_to_consider_dbz=None):
"""Finds echo top at each horizontal location.
"Echo top" is max height with reflectivity >= critical reflectivity.
M = number of rows (unique grid-point latitudes)
N = number of columns (unique grid-point longitudes)
:param unix_time_sec: Valid time.
:param spc_date_string: SPC date (format "yyyymmdd").
:param top_directory_name: Name of top-level directory with MYRORSS files.
:param critical_reflectivity_dbz: Critical reflectivity (used to define echo
top).
:param top_height_to_consider_m_asl: Top height level to consider (metres
above sea level).
:param lowest_refl_to_consider_dbz: Lowest reflectivity to consider in echo
top calculations. If None, will consider all reflectivities.
:return: echo_top_matrix_m_asl: M-by-N matrix of echo tops (metres above sea
level). Latitude increases down each column, and longitude increases to
the right along each row.
:return: grid_point_latitudes_deg: length-M numpy array with latitudes
(deg N) of grid points, sorted in ascending order.
:return: grid_point_longitudes_deg: length-N numpy array with longitudes
(deg E) of grid points, sorted in ascending order.
:return: metadata_dict: Dictionary created by
`myrorss_and_mrms_io.read_metadata_from_raw_file` for column-max
reflectivity.
"""
error_checking.assert_is_greater(critical_reflectivity_dbz, 0.)
error_checking.assert_is_greater(top_height_to_consider_m_asl, 0)
top_height_to_consider_m_asl = int(
numpy.round(top_height_to_consider_m_asl))
if lowest_refl_to_consider_dbz is None:
lowest_refl_to_consider_dbz = 0.
error_checking.assert_is_less_than(lowest_refl_to_consider_dbz,
critical_reflectivity_dbz)
grid_point_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_utils.REFL_NAME)
grid_point_heights_m_asl = grid_point_heights_m_asl[
grid_point_heights_m_asl <= top_height_to_consider_m_asl]
column_max_refl_file_name = myrorss_and_mrms_io.find_raw_file(
unix_time_sec=unix_time_sec,
spc_date_string=spc_date_string,
field_name=radar_utils.REFL_COLUMN_MAX_NAME,
data_source=radar_utils.MYRORSS_SOURCE_ID,
top_directory_name=top_directory_name)
num_grid_heights = len(grid_point_heights_m_asl)
single_height_refl_file_names = [''] * num_grid_heights
for k in range(num_grid_heights):
single_height_refl_file_names[k] = myrorss_and_mrms_io.find_raw_file(
unix_time_sec=unix_time_sec,
spc_date_string=spc_date_string,
field_name=radar_utils.REFL_NAME,
data_source=radar_utils.MYRORSS_SOURCE_ID,
top_directory_name=top_directory_name,
height_m_asl=grid_point_heights_m_asl[k])
print 'Reading "{0:s}" for echo-top calculation...'.format(
column_max_refl_file_name)
metadata_dict = myrorss_and_mrms_io.read_metadata_from_raw_file(
column_max_refl_file_name, data_source=radar_utils.MYRORSS_SOURCE_ID)
this_sparse_grid_table = (
myrorss_and_mrms_io.read_data_from_sparse_grid_file(
column_max_refl_file_name,
field_name_orig=metadata_dict[
myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG],
data_source=radar_utils.MYRORSS_SOURCE_ID,
sentinel_values=metadata_dict[radar_utils.SENTINEL_VALUE_COLUMN]))
(column_max_refl_matrix_dbz, grid_point_latitudes_deg,
grid_point_longitudes_deg) = radar_s2f.sparse_to_full_grid(
this_sparse_grid_table, metadata_dict)
num_grid_rows = len(grid_point_latitudes_deg)
num_grid_columns = len(grid_point_longitudes_deg)
linear_indices_to_consider = numpy.where(
numpy.reshape(column_max_refl_matrix_dbz, num_grid_rows *
num_grid_columns) >= critical_reflectivity_dbz)[0]
print(
'Echo-top calculation is needed at only {0:d}/{1:d} horizontal grid '
'points!').format(len(linear_indices_to_consider),
num_grid_rows * num_grid_columns)
echo_top_matrix_m_asl = numpy.full((num_grid_rows, num_grid_columns),
numpy.nan)
num_horiz_points_to_consider = len(linear_indices_to_consider)
if num_horiz_points_to_consider == 0:
return echo_top_matrix_m_asl
grid_rows_to_consider, grid_columns_to_consider = numpy.unravel_index(
linear_indices_to_consider, (num_grid_rows, num_grid_columns))
reflectivity_matrix_dbz = numpy.full(
(num_grid_heights, num_horiz_points_to_consider), numpy.nan)
for k in range(num_grid_heights):
print 'Reading "{0:s}" for echo-top calculation...'.format(
single_height_refl_file_names[k])
this_metadata_dict = myrorss_and_mrms_io.read_metadata_from_raw_file(
single_height_refl_file_names[k],
data_source=radar_utils.MYRORSS_SOURCE_ID)
this_sparse_grid_table = (
myrorss_and_mrms_io.read_data_from_sparse_grid_file(
single_height_refl_file_names[k],
field_name_orig=this_metadata_dict[
myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG],
data_source=radar_utils.MYRORSS_SOURCE_ID,
sentinel_values=this_metadata_dict[
radar_utils.SENTINEL_VALUE_COLUMN]))
this_reflectivity_matrix_dbz, _, _ = radar_s2f.sparse_to_full_grid(
this_sparse_grid_table,
this_metadata_dict,
ignore_if_below=lowest_refl_to_consider_dbz)
reflectivity_matrix_dbz[k, :] = this_reflectivity_matrix_dbz[
grid_rows_to_consider, grid_columns_to_consider]
print 'Computing echo tops at the {0:d} horizontal grid points...'.format(
num_horiz_points_to_consider)
for i in range(num_horiz_points_to_consider):
echo_top_matrix_m_asl[
grid_rows_to_consider[i], grid_columns_to_consider[i]] = (
radar_utils.get_echo_top_single_column(
reflectivities_dbz=reflectivity_matrix_dbz[:, i],
heights_m_asl=grid_point_heights_m_asl,
critical_reflectivity_dbz=critical_reflectivity_dbz))
return (numpy.flipud(echo_top_matrix_m_asl),
grid_point_latitudes_deg[::-1], grid_point_longitudes_deg,
metadata_dict)
AZIMUTHAL_RADAR_FIELD_NAMES = [
radar_utils.LOW_LEVEL_SHEAR_NAME, radar_utils.MID_LEVEL_SHEAR_NAME
]
DEFAULT_FIELDS_TO_REMOVE = [
radar_utils.ECHO_TOP_18DBZ_NAME, radar_utils.ECHO_TOP_50DBZ_NAME,
radar_utils.LOW_LEVEL_SHEAR_NAME, radar_utils.MID_LEVEL_SHEAR_NAME,
radar_utils.REFL_NAME, radar_utils.REFL_COLUMN_MAX_NAME,
radar_utils.MESH_NAME, radar_utils.REFL_0CELSIUS_NAME,
radar_utils.REFL_M10CELSIUS_NAME, radar_utils.REFL_M20CELSIUS_NAME,
radar_utils.REFL_LOWEST_ALTITUDE_NAME, radar_utils.SHI_NAME,
radar_utils.VIL_NAME
]
DEFAULT_REFL_HEIGHTS_TO_REMOVE_M_ASL = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID, field_name=radar_utils.REFL_NAME)
def unzip_1day_tar_file(
tar_file_name, field_names, spc_date_string, top_target_directory_name,
refl_heights_m_asl=None):
"""Unzips 1-day tar file (containing raw MYRORSS data for one SPC date).
:param tar_file_name: Path to input file.
:param field_names: 1-D list with names of radar fields.
:param spc_date_string: SPC date (format "yyyymmdd").
:param top_target_directory_name: Name of top-level directory for unzipped
MYRORSS files. This method will create a subdirectory therein for the
SPC date.
:param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres
above sea level).
def _extract_storm_images(num_image_rows, num_image_columns, rotate_grids,
rotated_grid_spacing_metres, radar_field_names,
refl_heights_m_agl, spc_date_string,
tarred_myrorss_dir_name, untarred_myrorss_dir_name,
top_tracking_dir_name, elevation_dir_name,
tracking_scale_metres2, target_name,
top_target_dir_name, top_output_dir_name):
"""Extracts storm-centered img for each field/height pair and storm object.
:param num_image_rows: See documentation at top of file.
:param num_image_columns: Same.
:param rotate_grids: Same.
:param rotated_grid_spacing_metres: Same.
:param radar_field_names: Same.
:param refl_heights_m_agl: Same.
:param spc_date_string: Same.
:param tarred_myrorss_dir_name: Same.
:param untarred_myrorss_dir_name: Same.
:param top_tracking_dir_name: Same.
:param elevation_dir_name: Same.
:param tracking_scale_metres2: Same.
:param target_name: Same.
:param top_target_dir_name: Same.
:param top_output_dir_name: Same.
"""
if target_name in ['', 'None']:
target_name = None
if target_name is not None:
target_param_dict = target_val_utils.target_name_to_params(target_name)
target_file_name = target_val_utils.find_target_file(
top_directory_name=top_target_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_string)
print('Reading data from: "{0:s}"...'.format(target_file_name))
target_dict = target_val_utils.read_target_values(
netcdf_file_name=target_file_name, target_names=[target_name])
print('\n')
refl_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_utils.REFL_NAME)
# Untar files with azimuthal shear.
az_shear_field_names = list(
set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES))
if len(az_shear_field_names):
az_shear_tar_file_name = (
'{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar').format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=az_shear_tar_file_name,
field_names=az_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name)
print(SEPARATOR_STRING)
# Untar files with other radar fields.
non_shear_field_names = list(
set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES))
if len(non_shear_field_names):
non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string)
myrorss_io.unzip_1day_tar_file(
tar_file_name=non_shear_tar_file_name,
field_names=non_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name,
refl_heights_m_asl=refl_heights_m_asl)
print(SEPARATOR_STRING)
# Read storm tracks for the given SPC date.
tracking_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_string,
source_name=tracking_utils.SEGMOTION_NAME,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2)[0]
storm_object_table = tracking_io.read_many_files(tracking_file_names)[
storm_images.STORM_COLUMNS_NEEDED]
print(SEPARATOR_STRING)
if target_name is not None:
print(('Removing storm objects without target values (variable = '
'"{0:s}")...').format(target_name))
these_indices = tracking_utils.find_storm_objects(
all_id_strings=storm_object_table[
tracking_utils.FULL_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.VALID_TIME_COLUMN].values.astype(int),
id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY],
times_to_keep_unix_sec=target_dict[
target_val_utils.VALID_TIMES_KEY],
allow_missing=False)
num_storm_objects_orig = len(storm_object_table.index)
storm_object_table = storm_object_table.iloc[these_indices]
num_storm_objects = len(storm_object_table.index)
print('Removed {0:d} of {1:d} storm objects!\n'.format(
num_storm_objects_orig - num_storm_objects,
num_storm_objects_orig))
# Extract storm-centered radar images.
storm_images.extract_storm_images_myrorss_or_mrms(
storm_object_table=storm_object_table,
radar_source=radar_utils.MYRORSS_SOURCE_ID,
top_radar_dir_name=untarred_myrorss_dir_name,
top_output_dir_name=top_output_dir_name,
elevation_dir_name=elevation_dir_name,
num_storm_image_rows=num_image_rows,
num_storm_image_columns=num_image_columns,
rotate_grids=rotate_grids,
rotated_grid_spacing_metres=rotated_grid_spacing_metres,
radar_field_names=radar_field_names,
reflectivity_heights_m_agl=refl_heights_m_agl)
print(SEPARATOR_STRING)
# Remove untarred MYRORSS files.
myrorss_io.remove_unzipped_data_1day(
spc_date_string=spc_date_string,
top_directory_name=untarred_myrorss_dir_name,
field_names=radar_field_names,
refl_heights_m_asl=refl_heights_m_asl)
print(SEPARATOR_STRING)
def _extract_storm_images(
num_image_rows, num_image_columns, rotate_grids,
rotated_grid_spacing_metres, radar_field_names, refl_heights_m_agl,
spc_date_string, first_time_string, last_time_string,
tarred_myrorss_dir_name, untarred_myrorss_dir_name,
top_tracking_dir_name, elevation_dir_name, tracking_scale_metres2,
target_name, top_target_dir_name, top_output_dir_name):
"""Extracts storm-centered img for each field/height pair and storm object.
:param num_image_rows: See documentation at top of file.
:param num_image_columns: Same.
:param rotate_grids: Same.
:param rotated_grid_spacing_metres: Same.
:param radar_field_names: Same.
:param refl_heights_m_agl: Same.
:param spc_date_string: Same.
:param first_time_string: Same.
:param last_time_string: Same.
:param tarred_myrorss_dir_name: Same.
:param untarred_myrorss_dir_name: Same.
:param top_tracking_dir_name: Same.
:param elevation_dir_name: Same.
:param tracking_scale_metres2: Same.
:param target_name: Same.
:param top_target_dir_name: Same.
:param top_output_dir_name: Same.
:raises: ValueError: if `first_time_string` and `last_time_string` have
different SPC dates.
"""
if elevation_dir_name in ['', 'None']:
elevation_dir_name = None
if elevation_dir_name is None:
host_name = socket.gethostname()
if 'casper' in host_name:
elevation_dir_name = '/glade/work/ryanlage/elevation'
else:
elevation_dir_name = '/condo/swatwork/ralager/elevation'
if spc_date_string in ['', 'None']:
first_time_unix_sec = time_conversion.string_to_unix_sec(
first_time_string, TIME_FORMAT)
last_time_unix_sec = time_conversion.string_to_unix_sec(
last_time_string, TIME_FORMAT)
first_spc_date_string = time_conversion.time_to_spc_date_string(
first_time_unix_sec)
last_spc_date_string = time_conversion.time_to_spc_date_string(
last_time_unix_sec)
if first_spc_date_string != last_spc_date_string:
error_string = (
'First ({0:s}) and last ({1:s}) times have different SPC dates.'
' This script can handle only one SPC date.'
).format(first_time_string, last_time_string)
raise ValueError(error_string)
spc_date_string = first_spc_date_string
else:
first_time_unix_sec = 0
last_time_unix_sec = int(1e12)
if tarred_myrorss_dir_name in ['', 'None']:
tarred_myrorss_dir_name = None
if target_name in ['', 'None']:
target_name = None
if target_name is not None:
target_param_dict = target_val_utils.target_name_to_params(target_name)
target_file_name = target_val_utils.find_target_file(
top_directory_name=top_target_dir_name,
event_type_string=target_param_dict[
target_val_utils.EVENT_TYPE_KEY],
spc_date_string=spc_date_string)
print('Reading data from: "{0:s}"...'.format(target_file_name))
target_dict = target_val_utils.read_target_values(
netcdf_file_name=target_file_name, target_names=[target_name]
)
print('\n')
refl_heights_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_utils.REFL_NAME)
# Untar files.
if tarred_myrorss_dir_name is not None:
az_shear_field_names = list(
set(radar_field_names) & set(ALL_AZ_SHEAR_FIELD_NAMES)
)
if len(az_shear_field_names) > 0:
az_shear_tar_file_name = (
'{0:s}/{1:s}/azimuthal_shear_only/{2:s}.tar'
).format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string
)
myrorss_io.unzip_1day_tar_file(
tar_file_name=az_shear_tar_file_name,
field_names=az_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name)
print(SEPARATOR_STRING)
non_shear_field_names = list(
set(radar_field_names) - set(ALL_AZ_SHEAR_FIELD_NAMES)
)
if len(non_shear_field_names) > 0:
non_shear_tar_file_name = '{0:s}/{1:s}/{2:s}.tar'.format(
tarred_myrorss_dir_name, spc_date_string[:4], spc_date_string
)
myrorss_io.unzip_1day_tar_file(
tar_file_name=non_shear_tar_file_name,
field_names=non_shear_field_names,
spc_date_string=spc_date_string,
top_target_directory_name=untarred_myrorss_dir_name,
refl_heights_m_asl=refl_heights_m_asl)
print(SEPARATOR_STRING)
# Read storm tracks for the given SPC date.
tracking_file_names = tracking_io.find_files_one_spc_date(
spc_date_string=spc_date_string,
source_name=tracking_utils.SEGMOTION_NAME,
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2
)[0]
file_times_unix_sec = numpy.array(
[tracking_io.file_name_to_time(f) for f in tracking_file_names],
dtype=int
)
good_indices = numpy.where(numpy.logical_and(
file_times_unix_sec >= first_time_unix_sec,
file_times_unix_sec <= last_time_unix_sec
))[0]
tracking_file_names = [tracking_file_names[k] for k in good_indices]
storm_object_table = tracking_io.read_many_files(
tracking_file_names
)[storm_images.STORM_COLUMNS_NEEDED]
print(SEPARATOR_STRING)
if target_name is not None:
print((
'Removing storm objects without target values (variable = '
'"{0:s}")...'
).format(target_name))
these_indices = tracking_utils.find_storm_objects(
all_id_strings=storm_object_table[
tracking_utils.FULL_ID_COLUMN].values.tolist(),
all_times_unix_sec=storm_object_table[
tracking_utils.VALID_TIME_COLUMN].values.astype(int),
id_strings_to_keep=target_dict[target_val_utils.FULL_IDS_KEY],
times_to_keep_unix_sec=target_dict[
target_val_utils.VALID_TIMES_KEY],
allow_missing=False)
num_storm_objects_orig = len(storm_object_table.index)
storm_object_table = storm_object_table.iloc[these_indices]
num_storm_objects = len(storm_object_table.index)
print('Removed {0:d} of {1:d} storm objects!\n'.format(
num_storm_objects_orig - num_storm_objects, num_storm_objects_orig
))
# Extract storm-centered radar images.
storm_images.extract_storm_images_myrorss_or_mrms(
storm_object_table=storm_object_table,
radar_source=radar_utils.MYRORSS_SOURCE_ID,
top_radar_dir_name=untarred_myrorss_dir_name,
top_output_dir_name=top_output_dir_name,
elevation_dir_name=elevation_dir_name,
num_storm_image_rows=num_image_rows,
num_storm_image_columns=num_image_columns, rotate_grids=rotate_grids,
rotated_grid_spacing_metres=rotated_grid_spacing_metres,
radar_field_names=radar_field_names,
reflectivity_heights_m_agl=refl_heights_m_agl)
print(SEPARATOR_STRING)
# Remove untarred MYRORSS files.
if tarred_myrorss_dir_name is not None:
myrorss_io.remove_unzipped_data_1day(
spc_date_string=spc_date_string,
top_directory_name=untarred_myrorss_dir_name,
field_names=radar_field_names,
refl_heights_m_asl=refl_heights_m_asl)
from gewittergefahr.gg_utils import grids
from gewittergefahr.gg_utils import radar_utils
from gewittergefahr.gg_utils import echo_classification as echo_classifn
TOLERANCE = 1e-6
# The following constants are used to test _estimate_melting_levels.
MELTING_LEVEL_LATITUDES_DEG = numpy.linspace(-90., 90., num=19)
MELTING_LEVEL_TIME_UNIX_SEC = 1541823287 # 041447 UTC 10 Nov 2018
MELTING_LEVELS_M_ASL = (echo_classifn.MELT_LEVEL_INTERCEPT_BY_MONTH_M_ASL[10] +
echo_classifn.MELT_LEVEL_SLOPE_BY_MONTH_M_DEG01[10] *
numpy.absolute(MELTING_LEVEL_LATITUDES_DEG))
# The following constants are used to test _neigh_metres_to_rowcol.
LARGE_GRID_HEIGHTS_M_ASL = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_utils.REFL_NAME)
LARGE_GRID_METADATA_DICT = {
echo_classifn.MIN_LATITUDE_KEY: 20.,
echo_classifn.LATITUDE_SPACING_KEY: 0.01,
echo_classifn.MIN_LONGITUDE_KEY: 230.,
echo_classifn.LONGITUDE_SPACING_KEY: 0.01,
echo_classifn.HEIGHTS_KEY: LARGE_GRID_HEIGHTS_M_ASL
}
THESE_LATITUDES_DEG, THESE_LONGITUDES_DEG = grids.get_latlng_grid_points(
min_latitude_deg=LARGE_GRID_METADATA_DICT[echo_classifn.MIN_LATITUDE_KEY],
min_longitude_deg=LARGE_GRID_METADATA_DICT[
echo_classifn.MIN_LONGITUDE_KEY],
lat_spacing_deg=LARGE_GRID_METADATA_DICT[
def evaluate_tracks(storm_object_table, top_myrorss_dir_name,
radar_field_name):
"""Evaluates a set of storm tracks, following Lakshmanan and Smith (2010).
T = number of storm tracks
:param storm_object_table: pandas DataFrame with storm objects. Should
contain columns listed in `storm_tracking_io.write_file`.
:param top_myrorss_dir_name: Name of top-level directory with MYRORSS data.
Files therein will be found by `myrorss_and_mrms_io.find_raw_file` and
read by `myrorss_and_mrms_io.read_data_from_sparse_grid_file`.
:param radar_field_name: Name of radar field to use in computing mismatch
error. Must be accepted by `radar_utils.check_field_name`.
:return: evaluation_dict: Dictionary with the following keys.
evaluation_dict['track_durations_sec']: length-T numpy array of storm
durations.
evaluation_dict['track_linearity_errors_metres']: length-T numpy array of
linearity errors. The "linearity error" for one track is the RMSE
(root mean square error) of Theil-Sen estimates over all time steps.
evaluation_dict['track_mismatch_errors']: length-T numpy array of mismatch
errors. The "mismatch error" is the standard deviation of X over all
time steps, where X = median value of `radar_field_name` inside the
storm.
evaluation_dict['mean_linearity_error_metres']: Mean linearity error. This
is the mean of trackwise linearity errors for tracks with duration >=
median duration.
evaluation_dict['mean_mismatch_error']: Mean mismatch error. This is the
mean of trackwise mismatch errors for tracks with duration >= median
duration.
evaluation_dict['radar_field_name']: Same as input (metadata).
"""
# Add x-y coordinates.
projection_object = projections.init_azimuthal_equidistant_projection(
central_latitude_deg=echo_top_tracking.CENTRAL_PROJ_LATITUDE_DEG,
central_longitude_deg=echo_top_tracking.CENTRAL_PROJ_LONGITUDE_DEG)
x_coords_metres, y_coords_metres = projections.project_latlng_to_xy(
latitudes_deg=storm_object_table[
tracking_utils.CENTROID_LATITUDE_COLUMN].values,
longitudes_deg=storm_object_table[
tracking_utils.CENTROID_LONGITUDE_COLUMN].values,
projection_object=projection_object,
false_easting_metres=0.,
false_northing_metres=0.)
storm_object_table = storm_object_table.assign(
**{
tracking_utils.CENTROID_X_COLUMN: x_coords_metres,
tracking_utils.CENTROID_Y_COLUMN: y_coords_metres
})
# Convert list of storm objects to list of tracks.
storm_track_table = tracking_utils.storm_objects_to_tracks(
storm_object_table)
# Fit Theil-Sen model to each track.
print(SEPARATOR_STRING)
storm_track_table = _fit_theil_sen_many_tracks(storm_track_table)
print(SEPARATOR_STRING)
# Compute storm durations.
num_tracks = len(storm_track_table.index)
track_durations_sec = numpy.full(num_tracks, -1, dtype=int)
for i in range(num_tracks):
this_start_time_unix_sec = numpy.min(
storm_track_table[tracking_utils.TRACK_TIMES_COLUMN].values[i])
this_end_time_unix_sec = numpy.max(
storm_track_table[tracking_utils.TRACK_TIMES_COLUMN].values[i])
track_durations_sec[i] = (this_end_time_unix_sec -
this_start_time_unix_sec)
for this_percentile_level in DURATION_PERCENTILE_LEVELS:
this_percentile_seconds = numpy.percentile(track_durations_sec,
this_percentile_level)
print('{0:d}th percentile of track durations = {1:.1f} seconds'.format(
int(numpy.round(this_percentile_level)), this_percentile_seconds))
print('\n')
for this_percentile_level in DURATION_PERCENTILE_LEVELS:
this_percentile_seconds = numpy.percentile(
track_durations_sec[track_durations_sec != 0],
this_percentile_level)
print(
('{0:d}th percentile of non-zero track durations = {1:.1f} seconds'
).format(int(numpy.round(this_percentile_level)),
this_percentile_seconds))
print(SEPARATOR_STRING)
median_duration_sec = numpy.median(
track_durations_sec[track_durations_sec != 0])
long_track_flags = track_durations_sec >= median_duration_sec
long_track_indices = numpy.where(long_track_flags)[0]
# Compute linearity error for each track.
track_linearity_errors_metres = numpy.full(num_tracks, numpy.nan)
for i in range(num_tracks):
if numpy.mod(i, 50) == 0:
print(('Have computed linearity error for {0:d} of {1:d} tracks...'
).format(i, num_tracks))
track_linearity_errors_metres[i] = _get_mean_ts_error_one_track(
storm_track_table=storm_track_table, row_index=i)
print('Have computed linearity error for all {0:d} tracks!'.format(
num_tracks))
good_indices = numpy.where(
numpy.logical_and(
long_track_flags,
track_linearity_errors_metres <= MAX_LINEARITY_ERROR_METRES))[0]
mean_linearity_error_metres = numpy.mean(
track_linearity_errors_metres[good_indices])
print('Mean linearity error = {0:.1f} metres'.format(
mean_linearity_error_metres))
print(SEPARATOR_STRING)
# Compute mismatch error for each track.
radar_statistic_table = (
radar_stats.get_storm_based_radar_stats_myrorss_or_mrms(
storm_object_table=storm_object_table,
top_radar_dir_name=top_myrorss_dir_name,
radar_metadata_dict_for_tracking=None,
statistic_names=[],
percentile_levels=numpy.array([50.]),
radar_field_names=[radar_field_name],
radar_source=radar_utils.MYRORSS_SOURCE_ID))
print(SEPARATOR_STRING)
radar_height_m_asl = radar_utils.get_valid_heights(
data_source=radar_utils.MYRORSS_SOURCE_ID,
field_name=radar_field_name)[0]
median_column_name = radar_stats.radar_field_and_percentile_to_column_name(
radar_field_name=radar_field_name,
radar_height_m_asl=radar_height_m_asl,
percentile_level=50.)
median_by_storm_object = radar_statistic_table[median_column_name]
track_mismatch_errors = numpy.full(num_tracks, numpy.nan)
for i in range(num_tracks):
these_object_indices = storm_track_table[
tracking_utils.OBJECT_INDICES_COLUMN].values[i]
if len(these_object_indices) < 2:
continue
track_mismatch_errors[i] = numpy.std(
median_by_storm_object[these_object_indices], ddof=1)
mean_mismatch_error = numpy.nanmean(
track_mismatch_errors[long_track_indices])
print('Mean mismatch error for "{0:s}" = {1:.4e}'.format(
radar_field_name, mean_mismatch_error))
return {
DURATIONS_KEY: track_durations_sec,
LINEARITY_ERRORS_KEY: track_linearity_errors_metres,
MISMATCH_ERRORS_KEY: track_mismatch_errors,
MEAN_LINEARITY_ERROR_KEY: mean_linearity_error_metres,
MEAN_MISMATCH_ERROR_KEY: mean_mismatch_error,
RADAR_FIELD_KEY: radar_field_name
}
