def find_prediction_file(top_directory_name, spc_date_string,
raise_error_if_missing=False):
"""Finds file with upconvnet predictions (reconstructed radar images).
:param top_directory_name: Name of top-level directory with upconvnet
predictions.
:param spc_date_string: SPC date (format "yyyymmdd").
:param raise_error_if_missing: Boolean flag. If file is missing and
`raise_error_if_missing = True`, this method will error out.
:return: prediction_file_name: Path to prediction file. If file is missing
and `raise_error_if_missing = False`, this will be the expected path.
:raises: ValueError: if file is missing and `raise_error_if_missing = True`.
"""
error_checking.assert_is_string(top_directory_name)
error_checking.assert_is_boolean(raise_error_if_missing)
time_conversion.spc_date_string_to_unix_sec(spc_date_string)
prediction_file_name = (
'{0:s}/{1:s}/{2:s}_{3:s}.p'
).format(
top_directory_name, spc_date_string[:4], PATHLESS_FILE_NAME_PREFIX,
spc_date_string
)
if raise_error_if_missing and not os.path.isfile(prediction_file_name):
error_string = 'Cannot find file. Expected at: "{0:s}"'.format(
prediction_file_name)
raise ValueError(error_string)
return prediction_file_name
def find_target_file(top_directory_name,
event_type_string,
spc_date_string,
raise_error_if_missing=True,
unix_time_sec=None):
"""Locates file with target values for either one time or one SPC date.
:param top_directory_name: Name of top-level directory with target files.
:param event_type_string: Event type (must be accepted by
`linkage.check_event_type`).
:param spc_date_string: SPC date (format "yyyymmdd").
:param raise_error_if_missing: Boolean flag. If file is missing and
`raise_error_if_missing = True`, this method will error out.
:param unix_time_sec: Valid time.
:return: target_file_name: Path to linkage file. If file is missing and
`raise_error_if_missing = False`, this will be the *expected* path.
:raises: ValueError: if file is missing and `raise_error_if_missing = True`.
"""
error_checking.assert_is_string(top_directory_name)
linkage.check_event_type(event_type_string)
error_checking.assert_is_boolean(raise_error_if_missing)
if unix_time_sec is None:
time_conversion.spc_date_string_to_unix_sec(spc_date_string)
if event_type_string == linkage.WIND_EVENT_STRING:
target_file_name = '{0:s}/{1:s}/wind_labels_{2:s}.nc'.format(
top_directory_name, spc_date_string[:4], spc_date_string)
else:
target_file_name = '{0:s}/{1:s}/tornado_labels_{2:s}.nc'.format(
top_directory_name, spc_date_string[:4], spc_date_string)
else:
spc_date_string = time_conversion.time_to_spc_date_string(
unix_time_sec)
valid_time_string = time_conversion.unix_sec_to_string(
unix_time_sec, TIME_FORMAT)
if event_type_string == linkage.WIND_EVENT_STRING:
target_file_name = '{0:s}/{1:s}/{2:s}/wind_labels_{3:s}.nc'.format(
top_directory_name, spc_date_string[:4], spc_date_string,
valid_time_string)
else:
target_file_name = (
'{0:s}/{1:s}/{2:s}/tornado_labels_{3:s}.nc').format(
top_directory_name, spc_date_string[:4], spc_date_string,
valid_time_string)
if raise_error_if_missing and not os.path.isfile(target_file_name):
error_string = 'Cannot find file. Expected at: "{0:s}"'.format(
target_file_name)
raise ValueError(error_string)
return target_file_name
def find_files_one_spc_date(
top_tracking_dir_name, tracking_scale_metres2, source_name,
spc_date_string, raise_error_if_missing=True):
"""Finds tracking files for the given SPC date.
:param top_tracking_dir_name: See doc for `_check_file_finding_args`.
:param tracking_scale_metres2: Same.
:param source_name: Same.
:param spc_date_string: SPC date (format "yyyymmdd").
:param raise_error_if_missing: Boolean flag. If no files are found and
`raise_error_if_missing = True`, this method will error out. If no
files are found and `raise_error_if_missing = False`, will return empty
list.
:return: tracking_file_names: 1-D list of paths to tracking files.
:return: glob_pattern: glob pattern used to find tracking files.
:raises: ValueError: if no files are found are
`raise_error_if_missing = True`.
"""
tracking_scale_metres2 = _check_file_finding_args(
top_tracking_dir_name=top_tracking_dir_name,
tracking_scale_metres2=tracking_scale_metres2, source_name=source_name,
raise_error_if_missing=raise_error_if_missing)
time_conversion.spc_date_string_to_unix_sec(spc_date_string)
directory_name = '{0:s}/{1:s}/{2:s}/scale_{3:d}m2'.format(
top_tracking_dir_name, spc_date_string[:4], spc_date_string,
tracking_scale_metres2
)
glob_pattern = (
'{0:s}/{1:s}_{2:s}_{3:s}{4:s}'
).format(
directory_name, FILE_NAME_PREFIX, source_name, FILE_NAME_TIME_REGEX,
FILE_EXTENSION
)
tracking_file_names = glob.glob(glob_pattern)
if raise_error_if_missing and not len(tracking_file_names):
error_string = 'Could not find any files with pattern: "{0:s}"'.format(
glob_pattern)
raise ValueError(error_string)
return tracking_file_names, glob_pattern
def remove_unzipped_data_1day(
spc_date_string, top_directory_name,
field_names=DEFAULT_FIELDS_TO_REMOVE,
refl_heights_m_agl=DEFAULT_REFL_HEIGHTS_TO_REMOVE_M_AGL):
"""Removes unzipped MYRORSS data for one SPC date.
Basically, this method cleans up after unzip_1day_tar_file.
:param spc_date_string: SPC date (format "yyyymmdd").
:param top_directory_name: Name of top-level directory with unzipped MYRORSS
files. This method will find the subdirectory in `top_directory_name`
for the given SPC date.
:param field_names: 1-D list with names of radar fields. Only these will be
deleted.
:param refl_heights_m_agl: 1-D integer numpy array of reflectivity heights
(metres above ground level).
"""
spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
spc_date_string)
field_to_heights_dict_m_agl = radar_io.field_and_height_arrays_to_dict(
field_names, refl_heights_m_agl=refl_heights_m_agl,
data_source=radar_io.MYRORSS_SOURCE_ID)
for this_field_name in field_to_heights_dict_m_agl.keys():
these_heights_m_agl = field_to_heights_dict_m_agl[this_field_name]
for this_height_m_agl in these_heights_m_agl:
example_file_name = radar_io.find_raw_file(
unix_time_sec=spc_date_unix_sec,
spc_date_unix_sec=spc_date_unix_sec, field_name=this_field_name,
height_m_agl=this_height_m_agl,
data_source=radar_io.MYRORSS_SOURCE_ID,
top_directory_name=top_directory_name,
raise_error_if_missing=False)
example_directory_name, _ = os.path.split(example_file_name)
directory_name_parts = example_directory_name.split('/')
remove_all_heights = False
if this_field_name == radar_io.REFL_NAME:
if (set(these_heights_m_agl) ==
set(DEFAULT_REFL_HEIGHTS_TO_REMOVE_M_AGL)):
remove_all_heights = True
dir_name_to_remove = '/'.join(directory_name_parts[:-1])
else:
dir_name_to_remove = '/'.join(directory_name_parts)
else:
dir_name_to_remove = '/'.join(directory_name_parts[:-1])
if os.path.isdir(dir_name_to_remove):
print 'Removing directory "{0:s}"...'.format(dir_name_to_remove)
shutil.rmtree(dir_name_to_remove, ignore_errors=True)
if remove_all_heights:
break
def find_raw_file(unix_time_sec,
spc_date_string,
field_name,
data_source,
top_directory_name,
height_m_asl=None,
raise_error_if_missing=True):
"""Finds raw file.
File should contain one field at one time step (e.g., MESH at 123502 UTC,
reflectivity at 500 m above sea level and 123502 UTC).
:param unix_time_sec: Valid time.
:param spc_date_string: SPC date (format "yyyymmdd").
:param field_name: Name of radar field in GewitterGefahr format.
:param data_source: Data source (string).
:param top_directory_name: Name of top-level directory with raw files.
:param height_m_asl: Radar height (metres above sea level).
:param raise_error_if_missing: Boolean flag. If True and file is missing,
this method will raise an error. If False and file is missing, will
return *expected* path to raw file.
:return: raw_file_name: Path to raw file.
:raises: ValueError: if raise_error_if_missing = True and file is missing.
"""
# Error-checking.
_ = time_conversion.spc_date_string_to_unix_sec(spc_date_string)
error_checking.assert_is_string(top_directory_name)
error_checking.assert_is_boolean(raise_error_if_missing)
relative_directory_name = get_relative_dir_for_raw_files(
field_name=field_name,
height_m_asl=height_m_asl,
data_source=data_source)
directory_name = '{0:s}/{1:s}/{2:s}/{3:s}'.format(top_directory_name,
spc_date_string[:4],
spc_date_string,
relative_directory_name)
pathless_file_name = _get_pathless_raw_file_name(unix_time_sec,
zipped=True)
raw_file_name = '{0:s}/{1:s}'.format(directory_name, pathless_file_name)
if raise_error_if_missing and not os.path.isfile(raw_file_name):
pathless_file_name = _get_pathless_raw_file_name(unix_time_sec,
zipped=False)
raw_file_name = '{0:s}/{1:s}'.format(directory_name,
pathless_file_name)
if raise_error_if_missing and not os.path.isfile(raw_file_name):
raise ValueError('Cannot find raw file. Expected at: "{0:s}"'.format(
raw_file_name))
return raw_file_name
def read_stats_from_xml(xml_file_name, spc_date_string):
"""Reads storm statistics from XML file.
:param xml_file_name: Path to input file.
:param spc_date_string: SPC date (format "yyyymmdd").
:return: stats_table: pandas DataFrame with the following columns.
stats_table.storm_id: String ID for storm cell.
stats_table.east_velocity_m_s01: Eastward velocity (m/s).
stats_table.north_velocity_m_s01: Northward velocity (m/s).
stats_table.age_sec: Age of storm cell (seconds).
"""
# Verification.
_ = time_conversion.spc_date_string_to_unix_sec(spc_date_string)
error_checking.assert_file_exists(xml_file_name)
xml_tree = _open_xml_file(xml_file_name)
storm_dict = {}
this_column_name = None
this_column_name_orig = None
this_column_values = None
for this_element in xml_tree.iter():
if this_element.tag == 'datacolumn':
if this_column_name_orig in XML_COLUMN_NAMES_ORIG:
storm_dict.update({this_column_name: this_column_values})
this_column_name_orig = this_element.attrib['name']
if this_column_name_orig in XML_COLUMN_NAMES_ORIG:
this_column_name = _xml_column_name_orig_to_new(
this_column_name_orig)
this_column_values = []
continue
if this_column_name_orig not in XML_COLUMN_NAMES_ORIG:
continue
if this_column_name == tracking_utils.STORM_ID_COLUMN:
this_column_values.append(this_element.attrib['value'])
elif this_column_name == tracking_utils.NORTH_VELOCITY_COLUMN:
this_column_values.append(-1 * float(this_element.attrib['value']))
elif this_column_name == tracking_utils.EAST_VELOCITY_COLUMN:
this_column_values.append(float(this_element.attrib['value']))
elif this_column_name == tracking_utils.AGE_COLUMN:
this_column_values.append(
int(numpy.round(float(this_element.attrib['value']))))
stats_table = pandas.DataFrame.from_dict(storm_dict)
storm_ids = _append_spc_date_to_storm_ids(
stats_table[tracking_utils.STORM_ID_COLUMN].values, spc_date_string)
stats_table = stats_table.assign(
**{tracking_utils.STORM_ID_COLUMN: storm_ids})
return tracking_utils.remove_nan_rows_from_dataframe(stats_table)
def find_processed_files_one_spc_date(top_processed_dir_name,
tracking_scale_metres2,
data_source,
spc_date_string,
raise_error_if_missing=True):
"""Finds processed files with valid time in one SPC date.
:param top_processed_dir_name: See doc for
`_check_input_args_for_file_finding`.
:param tracking_scale_metres2: Same.
:param data_source: Same.
:param spc_date_string: SPC date (format "yyyymmdd").
:param raise_error_if_missing: See doc for `find_processed_files_at_times`.
:return: processed_file_names: 1-D list of paths to processed tracking
files.
:return: glob_pattern: glob pattern used to find files.
:raises: ValueError: if no files are found and
`raise_error_if_missing = True`.
"""
tracking_scale_metres2 = _check_input_args_for_file_finding(
top_processed_dir_name=top_processed_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
data_source=data_source,
raise_error_if_missing=raise_error_if_missing)
time_conversion.spc_date_string_to_unix_sec(spc_date_string)
glob_pattern = (
'{0:s}/{1:s}/{2:s}/scale_{3:d}m2/{4:s}_{5:s}_{6:s}{7:s}').format(
top_processed_dir_name, spc_date_string[:4], spc_date_string,
tracking_scale_metres2, PREFIX_FOR_PATHLESS_FILE_NAMES,
data_source, TIME_FORMAT_IN_FILE_NAMES_AS_REGEX, FILE_EXTENSION)
processed_file_names = glob.glob(glob_pattern)
if raise_error_if_missing and not len(processed_file_names):
error_string = 'Could not find any files with pattern: "{0:s}"'.format(
glob_pattern)
raise ValueError(error_string)
return processed_file_names, glob_pattern
def find_raw_files_one_spc_date(spc_date_string,
field_name,
data_source,
top_directory_name,
height_m_asl=None,
raise_error_if_missing=True):
"""Finds raw files for one field and one SPC date.
:param spc_date_string: SPC date (format "yyyymmdd").
:param field_name: Name of radar field in GewitterGefahr format.
:param data_source: Data source (string).
:param top_directory_name: Name of top-level directory with raw files.
:param height_m_asl: Radar height (metres above sea level).
:param raise_error_if_missing: Boolean flag. If True and no files are
found, will raise error.
:return: raw_file_names: 1-D list of paths to raw files.
:raises: ValueError: if raise_error_if_missing = True and no files are
found.
"""
error_checking.assert_is_boolean(raise_error_if_missing)
example_time_unix_sec = time_conversion.spc_date_string_to_unix_sec(
spc_date_string)
example_file_name = find_raw_file(unix_time_sec=example_time_unix_sec,
spc_date_string=spc_date_string,
field_name=field_name,
data_source=data_source,
top_directory_name=top_directory_name,
height_m_asl=height_m_asl,
raise_error_if_missing=False)
example_directory_name, example_pathless_file_name = os.path.split(
example_file_name)
example_time_string = time_conversion.unix_sec_to_string(
example_time_unix_sec, TIME_FORMAT_SECONDS)
pathless_file_pattern = example_pathless_file_name.replace(
example_time_string, TIME_FORMAT_SECONDS_REGEX)
pathless_file_pattern = pathless_file_pattern.replace(
ZIPPED_FILE_EXTENSION, '*')
raw_file_pattern = '{0:s}/{1:s}'.format(example_directory_name,
pathless_file_pattern)
raw_file_names = glob.glob(raw_file_pattern)
if raise_error_if_missing and not raw_file_names:
error_string = (
'Could not find any files with the following pattern: {0:s}'
).format(raw_file_pattern)
raise ValueError(error_string)
return raw_file_names
def unzip_1day_tar_file(tar_file_name, spc_date_string, top_target_dir_name,
scales_to_extract_metres2):
"""Unzips tar file with segmotion output for one SPC date.
:param tar_file_name: Path to input file.
:param spc_date_string: SPC date (format "yyyymmdd").
:param top_target_dir_name: Name of top-level output directory.
:param scales_to_extract_metres2: 1-D numpy array of tracking scales to
extract.
:return: target_directory_name: Path to output directory. This will be
"<top_target_directory_name>/<yyyymmdd>", where <yyyymmdd> is the SPC
date.
"""
# Verification.
_ = time_conversion.spc_date_string_to_unix_sec(spc_date_string)
error_checking.assert_file_exists(tar_file_name)
error_checking.assert_is_greater_numpy_array(scales_to_extract_metres2, 0)
error_checking.assert_is_numpy_array(scales_to_extract_metres2,
num_dimensions=1)
scales_to_extract_metres2 = numpy.round(scales_to_extract_metres2).astype(
int)
num_scales_to_extract = len(scales_to_extract_metres2)
directory_names_to_unzip = []
for j in range(num_scales_to_extract):
this_relative_stats_dir_name = '{0:s}/{1:s}'.format(
spc_date_string,
_get_relative_stats_dir_physical_scale(
scales_to_extract_metres2[j]))
this_relative_polygon_dir_name = '{0:s}/{1:s}'.format(
spc_date_string,
_get_relative_polygon_dir_physical_scale(
scales_to_extract_metres2[j]))
directory_names_to_unzip.append(
this_relative_stats_dir_name.replace(spc_date_string + '/', ''))
directory_names_to_unzip.append(
this_relative_polygon_dir_name.replace(spc_date_string + '/', ''))
target_directory_name = '{0:s}/{1:s}/{2:s}'.format(top_target_dir_name,
spc_date_string[:4],
spc_date_string)
unzipping.unzip_tar(tar_file_name,
target_directory_name=target_directory_name,
file_and_dir_names_to_unzip=directory_names_to_unzip)
return target_directory_name
def find_local_polygon_file(unix_time_sec,
spc_date_string,
top_raw_directory_name,
tracking_scale_metres2,
raise_error_if_missing=True):
"""Finds polygon file on local machine.
This file should contain storm outlines (polygons) for one time step and one
tracking scale.
:param unix_time_sec: Valid time.
:param spc_date_string: SPC date (format "yyyymmdd").
:param top_raw_directory_name: Name of top-level directory with raw
segmotion files.
:param tracking_scale_metres2: Tracking scale.
:param raise_error_if_missing: Boolean flag. If True and file is missing,
this method will raise an error.
:return: polygon_file_name: Path to polygon file. If
raise_error_if_missing = False and file is missing, this will be the
*expected* path.
:raises: ValueError: if raise_error_if_missing = True and file is missing.
"""
# Verification.
_ = time_conversion.spc_date_string_to_unix_sec(spc_date_string)
error_checking.assert_is_string(top_raw_directory_name)
error_checking.assert_is_greater(tracking_scale_metres2, 0.)
error_checking.assert_is_boolean(raise_error_if_missing)
directory_name = '{0:s}/{1:s}/{2:s}/{3:s}'.format(
top_raw_directory_name, spc_date_string[:4], spc_date_string,
_get_relative_polygon_dir_physical_scale(tracking_scale_metres2))
pathless_file_name = _get_pathless_polygon_file_name(unix_time_sec,
zipped=True)
polygon_file_name = '{0:s}/{1:s}'.format(directory_name,
pathless_file_name)
if raise_error_if_missing and not os.path.isfile(polygon_file_name):
pathless_file_name = _get_pathless_polygon_file_name(unix_time_sec,
zipped=False)
polygon_file_name = '{0:s}/{1:s}'.format(directory_name,
pathless_file_name)
if raise_error_if_missing and not os.path.isfile(polygon_file_name):
raise ValueError('Cannot find polygon file. Expected at location: ' +
polygon_file_name)
return polygon_file_name
def find_files_for_smart_io(start_time_unix_sec=None,
start_spc_date_string=None,
end_time_unix_sec=None,
end_spc_date_string=None,
data_source=None,
tracking_scale_metres2=None,
top_input_dir_name=None,
top_output_dir_name=None):
"""Finds input, output, and temporary working files for smart IO.
N = number of SPC dates in period
T_i = number of time steps in the [i]th SPC date
:param start_time_unix_sec: Beginning of time period.
:param start_spc_date_string: SPC date at beginning of time period (format
"yyyymmdd").
:param end_time_unix_sec: End of time period.
:param end_spc_date_string: SPC date at end of time period (format
"yyyymmdd").
:param data_source: Source for input data (examples: "segmotion",
"probSevere").
:param tracking_scale_metres2: Tracking scale.
:param top_input_dir_name: Name of top-level directory for input files.
:param top_output_dir_name: Name of top-level directory for output files.
:return: file_dict: Dictionary with the following keys.
file_dict.spc_dates_unix_sec: length-N numpy array of SPC dates.
file_dict.temp_file_names: 1-D list of paths to temp files (will be used for
intermediate IO).
file_dict.input_file_names_by_spc_date: length-N list, where the [i]th
element is a 1-D list (length T_i) of paths to input files.
file_dict.output_file_names_by_spc_date: Same but for output files.
:raises: ValueError: if start_time_unix_sec is not part of the first SPC
date (determined by start_spc_date_unix_sec).
:raises: ValueError: if end_time_unix_sec is not part of the last SPC date
(determined by end_spc_date_unix_sec).
"""
if not time_conversion.is_time_in_spc_date(start_time_unix_sec,
start_spc_date_string):
start_time_string = time_conversion.unix_sec_to_string(
start_time_unix_sec, TIME_FORMAT_FOR_MESSAGES)
raise ValueError('Start time (' + start_time_string +
') is not in first SPC date (' +
start_spc_date_string + ').')
if not time_conversion.is_time_in_spc_date(end_time_unix_sec,
end_spc_date_string):
end_time_string = time_conversion.unix_sec_to_string(
end_time_unix_sec, TIME_FORMAT_FOR_MESSAGES)
raise ValueError('End time (' + end_time_string +
') is not in last SPC date (' + end_spc_date_string +
').')
error_checking.assert_is_greater(end_time_unix_sec, start_time_unix_sec)
start_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
start_spc_date_string)
end_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
end_spc_date_string)
num_spc_dates = int(1 + (end_spc_date_unix_sec - start_spc_date_unix_sec) /
DAYS_TO_SECONDS)
spc_dates_unix_sec = numpy.linspace(start_spc_date_unix_sec,
end_spc_date_unix_sec,
num=num_spc_dates,
dtype=int)
temp_file_names = [''] * num_spc_dates
input_file_names_by_spc_date = [['']] * num_spc_dates
output_file_names_by_spc_date = [['']] * num_spc_dates
for i in range(num_spc_dates):
spc_dates_unix_sec[i] = time_conversion.time_to_spc_date_unix_sec(
spc_dates_unix_sec[i])
temp_file_names[i] = tempfile.NamedTemporaryFile(delete=False).name
input_file_names_by_spc_date[i] = (
tracking_io.find_processed_files_one_spc_date(
spc_dates_unix_sec[i],
data_source=data_source,
top_processed_dir_name=top_input_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
raise_error_if_missing=True))
this_num_files = len(input_file_names_by_spc_date[i])
these_times_unix_sec = numpy.full(this_num_files, -1, dtype=int)
output_file_names_by_spc_date[i] = [''] * this_num_files
for j in range(this_num_files):
these_times_unix_sec[j] = tracking_io.processed_file_name_to_time(
input_file_names_by_spc_date[i][j])
output_file_names_by_spc_date[i][j] = (
tracking_io.find_processed_file(
unix_time_sec=these_times_unix_sec[j],
data_source=data_source,
spc_date_unix_sec=spc_dates_unix_sec[i],
top_processed_dir_name=top_output_dir_name,
tracking_scale_metres2=tracking_scale_metres2,
raise_error_if_missing=False))
if i == 0:
keep_time_indices = numpy.where(
these_times_unix_sec >= start_time_unix_sec)[0]
these_times_unix_sec = these_times_unix_sec[keep_time_indices]
input_file_names_by_spc_date[i] = [
input_file_names_by_spc_date[i][j] for j in keep_time_indices
]
output_file_names_by_spc_date[i] = [
output_file_names_by_spc_date[i][j] for j in keep_time_indices
]
if i == num_spc_dates - 1:
keep_time_indices = numpy.where(
these_times_unix_sec <= end_time_unix_sec)[0]
input_file_names_by_spc_date[i] = [
input_file_names_by_spc_date[i][j] for j in keep_time_indices
]
output_file_names_by_spc_date[i] = [
output_file_names_by_spc_date[i][j] for j in keep_time_indices
]
return {
SPC_DATES_KEY: spc_dates_unix_sec,
TEMP_FILE_NAMES_KEY: temp_file_names,
INPUT_FILE_NAMES_KEY: input_file_names_by_spc_date,
OUTPUT_FILE_NAMES_KEY: output_file_names_by_spc_date
}
def unzip_1day_tar_file(
tar_file_name, field_names, spc_date_string, top_target_directory_name,
refl_heights_m_asl=None):
"""Unzips 1-day tar file (containing raw MYRORSS data for one SPC date).
:param tar_file_name: Path to input file.
:param field_names: 1-D list with names of radar fields.
:param spc_date_string: SPC date (format "yyyymmdd").
:param top_target_directory_name: Name of top-level directory for unzipped
MYRORSS files. This method will create a subdirectory therein for the
SPC date.
:param refl_heights_m_asl: 1-D numpy array of reflectivity heights (metres
above sea level).
:return: target_directory_name: Path to output directory.
"""
# Verification.
_ = time_conversion.spc_date_string_to_unix_sec(spc_date_string)
error_checking.assert_is_string_list(field_names)
error_checking.assert_is_numpy_array(
numpy.asarray(field_names), num_dimensions=1)
error_checking.assert_is_string(top_target_directory_name)
# Put azimuthal-shear fields (which are allowed to be missing) at the end.
# This way, if the tar command errors out due to missing data, it will do so
# after unzipping all the non-missing data.
field_names_removed = []
for this_field_name in AZIMUTHAL_RADAR_FIELD_NAMES:
if this_field_name in field_names:
field_names.remove(this_field_name)
field_names_removed.append(this_field_name)
for this_field_name in field_names_removed:
field_names.append(this_field_name)
field_to_heights_dict_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_dict(
field_names=field_names, data_source=radar_utils.MYRORSS_SOURCE_ID,
refl_heights_m_asl=refl_heights_m_asl))
target_directory_name = '{0:s}/{1:s}/{2:s}'.format(
top_target_directory_name, spc_date_string[:4], spc_date_string
)
field_names = list(field_to_heights_dict_m_asl.keys())
directory_names_to_unzip = []
for this_field_name in field_names:
these_heights_m_asl = field_to_heights_dict_m_asl[this_field_name]
for this_height_m_asl in these_heights_m_asl:
directory_names_to_unzip.append(
myrorss_and_mrms_io.get_relative_dir_for_raw_files(
field_name=this_field_name,
data_source=radar_utils.MYRORSS_SOURCE_ID,
height_m_asl=this_height_m_asl))
unzipping.unzip_tar(
tar_file_name,
target_directory_name=target_directory_name,
file_and_dir_names_to_unzip=directory_names_to_unzip)
return target_directory_name
def get_echo_tops(
unix_time_sec,
spc_date_string,
top_directory_name,
critical_reflectivity_dbz,
top_height_to_consider_m_asl=DEFAULT_TOP_INPUT_HEIGHT_FOR_ECHO_TOPS_M_ASL,
lowest_refl_to_consider_dbz=None):
"""Finds echo top at each horizontal location.
"Echo top" is max height with reflectivity >= critical reflectivity.
M = number of rows (unique grid-point latitudes)
N = number of columns (unique grid-point longitudes)
:param unix_time_sec: Valid time.
:param spc_date_string: SPC date (format "yyyymmdd").
:param top_directory_name: Name of top-level directory with MYRORSS files.
:param critical_reflectivity_dbz: Critical reflectivity (used to define echo
top).
:param top_height_to_consider_m_asl: Top height level to consider (metres
above sea level).
:param lowest_refl_to_consider_dbz: Lowest reflectivity to consider in echo
top calculations. If None, will consider all reflectivities.
:return: echo_top_matrix_m_asl: M-by-N matrix of echo tops (metres above sea
level). Latitude increases down each column, and longitude increases to
the right along each row.
:return: grid_point_latitudes_deg: length-M numpy array with latitudes
(deg N) of grid points, sorted in ascending order.
:return: grid_point_longitudes_deg: length-N numpy array with longitudes
(deg E) of grid points, sorted in ascending order.
:return: metadata_dict: Dictionary created by
`radar_io.read_metadata_from_raw_file` for column-max reflectivity.
"""
error_checking.assert_is_greater(critical_reflectivity_dbz, 0.)
error_checking.assert_is_integer(top_height_to_consider_m_asl)
error_checking.assert_is_greater(top_height_to_consider_m_asl, 0)
if lowest_refl_to_consider_dbz is None:
lowest_refl_to_consider_dbz = 0.
error_checking.assert_is_less_than(lowest_refl_to_consider_dbz,
critical_reflectivity_dbz)
grid_point_heights_m_asl = radar_io.get_valid_heights_for_field(
radar_io.REFL_NAME, data_source=radar_io.MYRORSS_SOURCE_ID)
grid_point_heights_m_asl = grid_point_heights_m_asl[
grid_point_heights_m_asl <= top_height_to_consider_m_asl]
spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
spc_date_string)
column_max_refl_file_name = radar_io.find_raw_file(
unix_time_sec=unix_time_sec,
spc_date_unix_sec=spc_date_unix_sec,
field_name=radar_io.REFL_COLUMN_MAX_NAME,
data_source=radar_io.MYRORSS_SOURCE_ID,
top_directory_name=top_directory_name)
num_grid_heights = len(grid_point_heights_m_asl)
single_height_refl_file_names = [''] * num_grid_heights
for k in range(num_grid_heights):
single_height_refl_file_names[k] = radar_io.find_raw_file(
unix_time_sec=unix_time_sec,
spc_date_unix_sec=spc_date_unix_sec,
field_name=radar_io.REFL_NAME,
height_m_agl=grid_point_heights_m_asl[k],
data_source=radar_io.MYRORSS_SOURCE_ID,
top_directory_name=top_directory_name)
print 'Reading "{0:s}" for echo-top calculation...'.format(
column_max_refl_file_name)
metadata_dict = radar_io.read_metadata_from_raw_file(
column_max_refl_file_name, data_source=radar_io.MYRORSS_SOURCE_ID)
this_sparse_grid_table = radar_io.read_data_from_sparse_grid_file(
column_max_refl_file_name,
field_name_orig=metadata_dict[radar_io.FIELD_NAME_COLUMN_ORIG],
data_source=radar_io.MYRORSS_SOURCE_ID,
sentinel_values=metadata_dict[radar_io.SENTINEL_VALUE_COLUMN])
(column_max_refl_matrix_dbz, grid_point_latitudes_deg,
grid_point_longitudes_deg) = radar_s2f.sparse_to_full_grid(
this_sparse_grid_table, metadata_dict)
num_grid_rows = len(grid_point_latitudes_deg)
num_grid_columns = len(grid_point_longitudes_deg)
linear_indices_to_consider = numpy.where(
numpy.reshape(column_max_refl_matrix_dbz, num_grid_rows *
num_grid_columns) >= critical_reflectivity_dbz)[0]
print(
'Echo-top calculation is needed at only {0:d}/{1:d} horizontal grid '
'points!').format(len(linear_indices_to_consider),
num_grid_rows * num_grid_columns)
echo_top_matrix_m_asl = numpy.full((num_grid_rows, num_grid_columns),
numpy.nan)
num_horiz_points_to_consider = len(linear_indices_to_consider)
if num_horiz_points_to_consider == 0:
return echo_top_matrix_m_asl
grid_rows_to_consider, grid_columns_to_consider = numpy.unravel_index(
linear_indices_to_consider, (num_grid_rows, num_grid_columns))
reflectivity_matrix_dbz = numpy.full(
(num_grid_heights, num_horiz_points_to_consider), numpy.nan)
for k in range(num_grid_heights):
print 'Reading "{0:s}" for echo-top calculation...'.format(
single_height_refl_file_names[k])
this_metadata_dict = radar_io.read_metadata_from_raw_file(
single_height_refl_file_names[k],
data_source=radar_io.MYRORSS_SOURCE_ID)
this_sparse_grid_table = radar_io.read_data_from_sparse_grid_file(
single_height_refl_file_names[k],
field_name_orig=this_metadata_dict[
radar_io.FIELD_NAME_COLUMN_ORIG],
data_source=radar_io.MYRORSS_SOURCE_ID,
sentinel_values=this_metadata_dict[radar_io.SENTINEL_VALUE_COLUMN])
this_reflectivity_matrix_dbz, _, _ = radar_s2f.sparse_to_full_grid(
this_sparse_grid_table,
this_metadata_dict,
ignore_if_below=lowest_refl_to_consider_dbz)
reflectivity_matrix_dbz[k, :] = this_reflectivity_matrix_dbz[
grid_rows_to_consider, grid_columns_to_consider]
print 'Computing echo tops at the {0:d} horizontal grid points...'.format(
num_horiz_points_to_consider)
for i in range(num_horiz_points_to_consider):
echo_top_matrix_m_asl[
grid_rows_to_consider[i], grid_columns_to_consider[i]] = (
radar_utils.get_echo_top_single_column(
reflectivities_dbz=reflectivity_matrix_dbz[:, i],
heights_m_asl=grid_point_heights_m_asl,
critical_reflectivity_dbz=critical_reflectivity_dbz))
return (numpy.flipud(echo_top_matrix_m_asl),
grid_point_latitudes_deg[::-1], grid_point_longitudes_deg,
metadata_dict)
"""Joins tables with storm statistics and polygons.
:param stats_table: pandas DataFrame created by read_stats_from_xml.
:param polygon_table: pandas DataFrame created by read_polygons_from_netcdf
or `tracking_io.make_buffers_around_polygons`.
:return: storm_table: pandas DataFrame with columns from both stats_table
and polygon_table.
"""
return polygon_table.merge(stats_table,
on=tracking_io.STORM_ID_COLUMN,
how='inner')
if __name__ == '__main__':
SPC_DATE_UNIX_SEC = time_conversion.spc_date_string_to_unix_sec(
SPC_DATE_STRING)
STATS_TABLE = read_stats_from_xml(XML_FILE_NAME,
spc_date_unix_sec=SPC_DATE_UNIX_SEC)
print STATS_TABLE
METADATA_DICT = radar_io.read_metadata_from_raw_file(
NETCDF_FILE_NAME, data_source=radar_io.MYRORSS_SOURCE_ID)
POLYGON_TABLE = read_polygons_from_netcdf(
NETCDF_FILE_NAME,
metadata_dict=METADATA_DICT,
spc_date_unix_sec=SPC_DATE_UNIX_SEC,
tracking_start_time_unix_sec=TRACKING_START_TIME_UNIX_SEC,
tracking_end_time_unix_sec=TRACKING_END_TIME_UNIX_SEC)
print POLYGON_TABLE
def find_many_raw_files(
desired_times_unix_sec,
spc_date_strings,
data_source,
field_names,
top_directory_name,
reflectivity_heights_m_asl=None,
max_time_offset_for_az_shear_sec=DEFAULT_MAX_TIME_OFFSET_FOR_AZ_SHEAR_SEC,
max_time_offset_for_non_shear_sec=DEFAULT_MAX_TIME_OFFSET_FOR_NON_SHEAR_SEC
):
"""Finds raw file for each field/height pair and time step.
N = number of input times
T = number of unique input times
F = number of field/height pairs
:param desired_times_unix_sec: length-N numpy array with desired valid
times.
:param spc_date_strings: length-N list of corresponding SPC dates (format
"yyyymmdd").
:param data_source: Data source ("myrorss" or "mrms").
:param field_names: 1-D list of field names.
:param top_directory_name: Name of top-level directory with radar data from
the given source.
:param reflectivity_heights_m_asl: 1-D numpy array of heights (metres above
sea level) for the field "reflectivity_dbz". If "reflectivity_dbz" is
not in `field_names`, leave this as None.
:param max_time_offset_for_az_shear_sec: Max time offset (between desired
and actual valid time) for azimuthal-shear fields.
:param max_time_offset_for_non_shear_sec: Max time offset (between desired
and actual valid time) for non-azimuthal-shear fields.
:return: file_dictionary: Dictionary with the following keys.
file_dictionary['radar_file_name_matrix']: T-by-F numpy array of paths to
raw files.
file_dictionary['unique_times_unix_sec']: length-T numpy array of unique
valid times.
file_dictionary['spc_date_strings_for_unique_times']: length-T numpy array
of corresponding SPC dates.
file_dictionary['field_name_by_pair']: length-F list of field names.
file_dictionary['height_by_pair_m_asl']: length-F numpy array of heights
(metres above sea level).
"""
field_name_by_pair, height_by_pair_m_asl = (
myrorss_and_mrms_utils.fields_and_refl_heights_to_pairs(
field_names=field_names,
data_source=data_source,
refl_heights_m_asl=reflectivity_heights_m_asl))
num_fields = len(field_name_by_pair)
error_checking.assert_is_integer_numpy_array(desired_times_unix_sec)
error_checking.assert_is_numpy_array(desired_times_unix_sec,
num_dimensions=1)
num_times = len(desired_times_unix_sec)
error_checking.assert_is_string_list(spc_date_strings)
error_checking.assert_is_numpy_array(numpy.array(spc_date_strings),
exact_dimensions=numpy.array(
[num_times]))
spc_dates_unix_sec = numpy.array([
time_conversion.spc_date_string_to_unix_sec(s)
for s in spc_date_strings
])
time_matrix = numpy.hstack(
(numpy.reshape(desired_times_unix_sec, (num_times, 1)),
numpy.reshape(spc_dates_unix_sec, (num_times, 1))))
unique_time_matrix = numpy.vstack(
{tuple(this_row)
for this_row in time_matrix}).astype(int)
unique_times_unix_sec = unique_time_matrix[:, 0]
spc_dates_at_unique_times_unix_sec = unique_time_matrix[:, 1]
sort_indices = numpy.argsort(unique_times_unix_sec)
unique_times_unix_sec = unique_times_unix_sec[sort_indices]
spc_dates_at_unique_times_unix_sec = spc_dates_at_unique_times_unix_sec[
sort_indices]
num_unique_times = len(unique_times_unix_sec)
radar_file_name_matrix = numpy.full((num_unique_times, num_fields),
'',
dtype=object)
for i in range(num_unique_times):
this_spc_date_string = time_conversion.time_to_spc_date_string(
spc_dates_at_unique_times_unix_sec[i])
for j in range(num_fields):
if field_name_by_pair[j] in AZIMUTHAL_SHEAR_FIELD_NAMES:
this_max_time_offset_sec = max_time_offset_for_az_shear_sec
this_raise_error_flag = False
else:
this_max_time_offset_sec = max_time_offset_for_non_shear_sec
this_raise_error_flag = True
if this_max_time_offset_sec == 0:
radar_file_name_matrix[i, j] = find_raw_file(
unix_time_sec=unique_times_unix_sec[i],
spc_date_string=this_spc_date_string,
field_name=field_name_by_pair[j],
data_source=data_source,
top_directory_name=top_directory_name,
height_m_asl=height_by_pair_m_asl[j],
raise_error_if_missing=this_raise_error_flag)
else:
radar_file_name_matrix[i, j] = find_raw_file_inexact_time(
desired_time_unix_sec=unique_times_unix_sec[i],
spc_date_string=this_spc_date_string,
field_name=field_name_by_pair[j],
data_source=data_source,
top_directory_name=top_directory_name,
height_m_asl=height_by_pair_m_asl[j],
max_time_offset_sec=this_max_time_offset_sec,
raise_error_if_missing=this_raise_error_flag)
if radar_file_name_matrix[i, j] is None:
this_time_string = time_conversion.unix_sec_to_string(
unique_times_unix_sec[i], TIME_FORMAT_FOR_LOG_MESSAGES)
warning_string = (
'Cannot find file for "{0:s}" at {1:d} metres ASL and '
'{2:s}.').format(field_name_by_pair[j],
int(height_by_pair_m_asl[j]),
this_time_string)
warnings.warn(warning_string)
return {
RADAR_FILE_NAMES_KEY: radar_file_name_matrix,
UNIQUE_TIMES_KEY: unique_times_unix_sec,
SPC_DATES_AT_UNIQUE_TIMES_KEY: spc_dates_at_unique_times_unix_sec,
FIELD_NAME_BY_PAIR_KEY: field_name_by_pair,
HEIGHT_BY_PAIR_KEY: numpy.round(height_by_pair_m_asl).astype(int)
}
def test_spc_date_string_to_unix_sec(self):
"""Ensures correct output from spc_date_string_to_unix_sec."""
this_spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
SPC_DATE_STRING)
self.assertTrue(this_spc_date_unix_sec == SPC_DATE_UNIX_SEC)
def find_raw_file_inexact_time(desired_time_unix_sec,
spc_date_string,
field_name,
data_source,
top_directory_name,
height_m_asl=None,
max_time_offset_sec=None,
raise_error_if_missing=False):
"""Finds raw file at inexact time.
If you know the exact valid time, use `find_raw_file`.
:param desired_time_unix_sec: Desired valid time.
:param spc_date_string: SPC date (format "yyyymmdd").
:param field_name: Field name in GewitterGefahr format.
:param data_source: Data source (string).
:param top_directory_name: Name of top-level directory with raw files.
:param height_m_asl: Radar height (metres above sea level).
:param max_time_offset_sec: Maximum offset between actual and desired valid
time.
For example, if `desired_time_unix_sec` is 162933 UTC 5 Jan 2018 and
`max_time_offset_sec` = 60, this method will look for az-shear at valid
times from 162833...163033 UTC 5 Jan 2018.
If None, this defaults to `DEFAULT_MAX_TIME_OFFSET_FOR_AZ_SHEAR_SEC` for
azimuthal-shear fields and `DEFAULT_MAX_TIME_OFFSET_FOR_NON_SHEAR_SEC` for
all other fields.
:param raise_error_if_missing: Boolean flag. If no file is found and
raise_error_if_missing = True, this method will error out. If no file
is found and raise_error_if_missing = False, will return None.
:return: raw_file_name: Path to raw file.
:raises: ValueError: if no file is found and raise_error_if_missing = True.
"""
# Error-checking.
error_checking.assert_is_integer(desired_time_unix_sec)
_ = time_conversion.spc_date_string_to_unix_sec(spc_date_string)
error_checking.assert_is_boolean(raise_error_if_missing)
radar_utils.check_field_name(field_name)
if max_time_offset_sec is None:
if field_name in AZIMUTHAL_SHEAR_FIELD_NAMES:
max_time_offset_sec = DEFAULT_MAX_TIME_OFFSET_FOR_AZ_SHEAR_SEC
else:
max_time_offset_sec = DEFAULT_MAX_TIME_OFFSET_FOR_NON_SHEAR_SEC
error_checking.assert_is_integer(max_time_offset_sec)
error_checking.assert_is_greater(max_time_offset_sec, 0)
first_allowed_minute_unix_sec = numpy.round(
int(
rounder.floor_to_nearest(
float(desired_time_unix_sec - max_time_offset_sec),
MINUTES_TO_SECONDS)))
last_allowed_minute_unix_sec = numpy.round(
int(
rounder.floor_to_nearest(
float(desired_time_unix_sec + max_time_offset_sec),
MINUTES_TO_SECONDS)))
allowed_minutes_unix_sec = time_periods.range_and_interval_to_list(
start_time_unix_sec=first_allowed_minute_unix_sec,
end_time_unix_sec=last_allowed_minute_unix_sec,
time_interval_sec=MINUTES_TO_SECONDS,
include_endpoint=True).astype(int)
relative_directory_name = get_relative_dir_for_raw_files(
field_name=field_name,
data_source=data_source,
height_m_asl=height_m_asl)
raw_file_names = []
for this_time_unix_sec in allowed_minutes_unix_sec:
this_pathless_file_pattern = _get_pathless_raw_file_pattern(
this_time_unix_sec)
this_file_pattern = '{0:s}/{1:s}/{2:s}/{3:s}/{4:s}'.format(
top_directory_name, spc_date_string[:4], spc_date_string,
relative_directory_name, this_pathless_file_pattern)
raw_file_names += glob.glob(this_file_pattern)
file_times_unix_sec = []
for this_raw_file_name in raw_file_names:
file_times_unix_sec.append(raw_file_name_to_time(this_raw_file_name))
if len(file_times_unix_sec):
file_times_unix_sec = numpy.array(file_times_unix_sec)
time_differences_sec = numpy.absolute(file_times_unix_sec -
desired_time_unix_sec)
nearest_index = numpy.argmin(time_differences_sec)
min_time_diff_sec = time_differences_sec[nearest_index]
else:
min_time_diff_sec = numpy.inf
if min_time_diff_sec > max_time_offset_sec:
if raise_error_if_missing:
desired_time_string = time_conversion.unix_sec_to_string(
desired_time_unix_sec, TIME_FORMAT_FOR_LOG_MESSAGES)
error_string = (
'Could not find "{0:s}" file within {1:d} seconds of {2:s}.'
).format(field_name, max_time_offset_sec, desired_time_string)
raise ValueError(error_string)
return None
return raw_file_names[nearest_index]
def read_polygons_from_netcdf(netcdf_file_name,
metadata_dict,
spc_date_string,
tracking_start_time_unix_sec,
tracking_end_time_unix_sec,
raise_error_if_fails=True):
"""Reads storm polygons (outlines of storm cells) from NetCDF file.
P = number of grid points in storm cell (different for each storm cell)
V = number of vertices in storm polygon (different for each storm cell)
If file cannot be opened, returns None.
:param netcdf_file_name: Path to input file.
:param metadata_dict: Dictionary with metadata for NetCDF file, created by
`myrorss_and_mrms_io.read_metadata_from_raw_file`.
:param spc_date_string: SPC date (format "yyyymmdd").
:param tracking_start_time_unix_sec: Start time for tracking period. This
can be found by `get_start_end_times_for_spc_date`.
:param tracking_end_time_unix_sec: End time for tracking period. This can
be found by `get_start_end_times_for_spc_date`.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error.
:return: polygon_table: If file cannot be opened and raise_error_if_fails =
False, this is None. Otherwise, it is a pandas DataFrame with the
following columns.
polygon_table.storm_id: String ID for storm cell.
polygon_table.unix_time_sec: Time in Unix format.
polygon_table.spc_date_unix_sec: SPC date in Unix format.
polygon_table.tracking_start_time_unix_sec: Start time for tracking period.
polygon_table.tracking_end_time_unix_sec: End time for tracking period.
polygon_table.centroid_lat_deg: Latitude at centroid of storm cell (deg N).
polygon_table.centroid_lng_deg: Longitude at centroid of storm cell (deg E).
polygon_table.grid_point_latitudes_deg: length-P numpy array with latitudes
(deg N) of grid points in storm cell.
polygon_table.grid_point_longitudes_deg: length-P numpy array with
longitudes (deg E) of grid points in storm cell.
polygon_table.grid_point_rows: length-P numpy array with row indices (all
integers) of grid points in storm cell.
polygon_table.grid_point_columns: length-P numpy array with column indices
(all integers) of grid points in storm cell.
polygon_table.polygon_object_latlng: Instance of `shapely.geometry.Polygon`
with vertices in lat-long coordinates.
polygon_table.polygon_object_rowcol: Instance of `shapely.geometry.Polygon`
with vertices in row-column coordinates.
"""
error_checking.assert_file_exists(netcdf_file_name)
error_checking.assert_is_integer(tracking_start_time_unix_sec)
error_checking.assert_is_not_nan(tracking_start_time_unix_sec)
error_checking.assert_is_integer(tracking_end_time_unix_sec)
error_checking.assert_is_not_nan(tracking_end_time_unix_sec)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
storm_id_var_name = metadata_dict[radar_utils.FIELD_NAME_COLUMN]
storm_id_var_name_orig = metadata_dict[
myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG]
num_values = len(
netcdf_dataset.variables[myrorss_and_mrms_io.GRID_ROW_COLUMN_ORIG])
if num_values == 0:
sparse_grid_dict = {
myrorss_and_mrms_io.GRID_ROW_COLUMN: numpy.array([], dtype=int),
myrorss_and_mrms_io.GRID_COLUMN_COLUMN: numpy.array([], dtype=int),
myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN: numpy.array([],
dtype=int),
storm_id_var_name: numpy.array([], dtype=int)
}
else:
sparse_grid_dict = {
myrorss_and_mrms_io.GRID_ROW_COLUMN:
netcdf_dataset.variables[myrorss_and_mrms_io.GRID_ROW_COLUMN_ORIG]
[:],
myrorss_and_mrms_io.GRID_COLUMN_COLUMN:
netcdf_dataset.variables[
myrorss_and_mrms_io.GRID_COLUMN_COLUMN_ORIG][:],
myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN:
netcdf_dataset.variables[
myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN_ORIG][:],
storm_id_var_name:
netcdf_dataset.variables[storm_id_var_name_orig][:]
}
netcdf_dataset.close()
sparse_grid_table = pandas.DataFrame.from_dict(sparse_grid_dict)
numeric_storm_id_matrix, _, _ = (radar_s2f.sparse_to_full_grid(
sparse_grid_table, metadata_dict))
polygon_table = _storm_id_matrix_to_coord_lists(numeric_storm_id_matrix)
num_storms = len(polygon_table.index)
unix_times_sec = numpy.full(num_storms,
metadata_dict[radar_utils.UNIX_TIME_COLUMN],
dtype=int)
spc_date_unix_sec = time_conversion.spc_date_string_to_unix_sec(
spc_date_string)
spc_dates_unix_sec = numpy.full(num_storms, spc_date_unix_sec, dtype=int)
tracking_start_times_unix_sec = numpy.full(num_storms,
tracking_start_time_unix_sec,
dtype=int)
tracking_end_times_unix_sec = numpy.full(num_storms,
tracking_end_time_unix_sec,
dtype=int)
storm_ids = _append_spc_date_to_storm_ids(
polygon_table[tracking_utils.STORM_ID_COLUMN].values, spc_date_string)
simple_array = numpy.full(num_storms, numpy.nan)
object_array = numpy.full(num_storms, numpy.nan, dtype=object)
nested_array = polygon_table[[
tracking_utils.STORM_ID_COLUMN, tracking_utils.STORM_ID_COLUMN
]].values.tolist()
argument_dict = {
tracking_utils.STORM_ID_COLUMN: storm_ids,
tracking_utils.TIME_COLUMN: unix_times_sec,
tracking_utils.SPC_DATE_COLUMN: spc_dates_unix_sec,
tracking_utils.TRACKING_START_TIME_COLUMN:
tracking_start_times_unix_sec,
tracking_utils.TRACKING_END_TIME_COLUMN: tracking_end_times_unix_sec,
tracking_utils.CENTROID_LAT_COLUMN: simple_array,
tracking_utils.CENTROID_LNG_COLUMN: simple_array,
tracking_utils.GRID_POINT_LAT_COLUMN: nested_array,
tracking_utils.GRID_POINT_LNG_COLUMN: nested_array,
tracking_utils.POLYGON_OBJECT_LATLNG_COLUMN: object_array,
tracking_utils.POLYGON_OBJECT_ROWCOL_COLUMN: object_array
}
polygon_table = polygon_table.assign(**argument_dict)
for i in range(num_storms):
these_vertex_rows, these_vertex_columns = (
polygons.grid_points_in_poly_to_vertices(
polygon_table[tracking_utils.GRID_POINT_ROW_COLUMN].values[i],
polygon_table[
tracking_utils.GRID_POINT_COLUMN_COLUMN].values[i]))
# these_vertex_rows, these_vertex_columns = (
# polygons.grid_points_in_poly_to_vertices(
# metadata_dict[radar_utils.NUM_LAT_COLUMN] -
# polygon_table[tracking_utils.GRID_POINT_ROW_COLUMN].values[i],
# polygon_table[
# tracking_utils.GRID_POINT_COLUMN_COLUMN].values[i]))
#
# these_vertex_rows = (
# metadata_dict[radar_utils.NUM_LAT_COLUMN] - these_vertex_rows)
(polygon_table[tracking_utils.GRID_POINT_ROW_COLUMN].values[i],
polygon_table[tracking_utils.GRID_POINT_COLUMN_COLUMN].values[i]) = (
polygons.simple_polygon_to_grid_points(these_vertex_rows,
these_vertex_columns))
(polygon_table[tracking_utils.GRID_POINT_LAT_COLUMN].values[i],
polygon_table[tracking_utils.GRID_POINT_LNG_COLUMN].values[i]
) = (radar_utils.rowcol_to_latlng(
polygon_table[tracking_utils.GRID_POINT_ROW_COLUMN].values[i],
polygon_table[tracking_utils.GRID_POINT_COLUMN_COLUMN].values[i],
nw_grid_point_lat_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN]))
these_vertex_lat_deg, these_vertex_lng_deg = (
radar_utils.rowcol_to_latlng(
these_vertex_rows,
these_vertex_columns,
nw_grid_point_lat_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN]))
(polygon_table[tracking_utils.CENTROID_LAT_COLUMN].values[i],
polygon_table[tracking_utils.CENTROID_LNG_COLUMN].values[i]
) = geodetic_utils.get_latlng_centroid(
latitudes_deg=these_vertex_lat_deg,
longitudes_deg=these_vertex_lng_deg)
polygon_table[
tracking_utils.POLYGON_OBJECT_ROWCOL_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(
these_vertex_columns, these_vertex_rows))
polygon_table[
tracking_utils.POLYGON_OBJECT_LATLNG_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(
these_vertex_lng_deg, these_vertex_lat_deg))
return polygon_table
def _run(top_gridrad_dir_name, first_spc_date_string, last_spc_date_string):
"""Makes LaTeX table with GridRad days and domains.
This is effectively the main method.
:param top_gridrad_dir_name: See documentation at top of file.
:param first_spc_date_string: Same.
:param last_spc_date_string: Same.
"""
all_spc_date_strings = time_conversion.get_spc_dates_in_range(
first_spc_date_string=first_spc_date_string,
last_spc_date_string=last_spc_date_string)
nice_date_strings = []
latitude_strings = []
longitude_strings = []
for this_spc_date_string in all_spc_date_strings:
these_limits_deg = _find_domain_for_date(
top_gridrad_dir_name=top_gridrad_dir_name,
spc_date_string=this_spc_date_string
)
if these_limits_deg is None:
continue
print(SEPARATOR_STRING)
this_time_unix_sec = time_conversion.spc_date_string_to_unix_sec(
this_spc_date_string
)
this_nice_date_string = time_conversion.unix_sec_to_string(
this_time_unix_sec, NICE_TIME_FORMAT
)
nice_date_strings.append(this_nice_date_string)
latitude_strings.append('{0:.1f}-{1:.1f}'.format(
these_limits_deg[0], these_limits_deg[1]
))
longitude_strings.append('{0:.1f}-{1:.1f}'.format(
these_limits_deg[3], these_limits_deg[2]
))
table_string = ''
for i in range(len(nice_date_strings)):
if i != 0:
if numpy.mod(i, 2) == 0:
table_string += ' \\\\\n\t\t\t'
else:
table_string += ' & '
table_string += '{0:s}, {1:s}'.format(
nice_date_strings[i], latitude_strings[i]
)
# table_string += ' $^{\\circ}$N'
table_string += ', {0:s}'.format(longitude_strings[i])
# table_string += ' $^{\\circ}$W'
print(table_string)
