def read_target_values(netcdf_file_name, target_name):
"""Reads target values from NetCDF file.
N = number of storm objects
:param netcdf_file_name: Path to input file.
:param target_name: Name of target variable.
:return: storm_label_dict: Dictionary with the following keys.
storm_label_dict['storm_ids']: length-N list of storm IDs.
storm_label_dict['valid_times_unix_sec']: length-N numpy array of valid
times.
storm_label_dict['target_values']: length-N numpy array with values of
`target_name`.
"""
error_checking.assert_is_string(target_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name=netcdf_file_name,
raise_error_if_fails=True)
storm_ids = netCDF4.chartostring(
netcdf_dataset.variables[STORM_IDS_KEY][:])
valid_times_unix_sec = numpy.array(
netcdf_dataset.variables[VALID_TIMES_KEY][:], dtype=int)
target_values = numpy.array(netcdf_dataset.variables[target_name][:],
dtype=int)
netcdf_dataset.close()
return {
STORM_IDS_KEY: [str(s) for s in storm_ids],
VALID_TIMES_KEY: valid_times_unix_sec,
TARGET_VALUES_KEY: target_values
}
def read_field_from_full_grid_file(netcdf_file_name,
field_name=None,
metadata_dict=None,
raise_error_if_fails=True):
"""Reads one radar field from full-grid (not sparse-grid) file.
This file should contain all radar variables for one time step.
M = number of rows (unique grid-point latitudes)
N = number of columns (unique grid-point longitudes)
H = number of height levels (unique grid-point heights)
:param netcdf_file_name: Path to input file.
:param field_name: Name of radar field.
:param metadata_dict: Dictionary created by
read_metadata_from_full_grid_file.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error. If False and file cannot be
opened, will return None for all output variables.
:return: field_matrix: H-by-M-by-N numpy array with values of radar field.
:return: grid_point_heights_m_asl: length-H numpy array of height levels
(integer metres above sea level). If array is increasing (decreasing),
height increases (decreases) with the first index of field_matrix.
:return: grid_point_latitudes_deg: length-M numpy array of grid-point
latitudes (deg N). If array is increasing (decreasing), latitude
increases (decreases) with the second index of field_matrix.
:return: grid_point_longitudes_deg: length-N numpy array of grid-point
longitudes (deg N). If array is increasing (decreasing), latitude
increases (decreases) with the third index of field_matrix.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None, None, None, None
field_name_orig = radar_utils.field_name_new_to_orig(
field_name=field_name, data_source_name=radar_utils.GRIDRAD_SOURCE_ID)
field_matrix = numpy.array(
netcdf_dataset.variables[field_name_orig][0, :, :, :])
grid_point_latitudes_deg = numpy.array(
netcdf_dataset.variables[LATITUDE_NAME_ORIG])
grid_point_longitudes_deg = lng_conversion.convert_lng_positive_in_west(
numpy.array(netcdf_dataset.variables[LONGITUDE_NAME_ORIG]))
_check_grid_points(grid_point_latitudes_deg=grid_point_latitudes_deg,
grid_point_longitudes_deg=grid_point_longitudes_deg,
metadata_dict=metadata_dict)
grid_point_heights_m_asl = KM_TO_METRES * numpy.array(
netcdf_dataset.variables[HEIGHT_NAME_ORIG])
grid_point_heights_m_asl = numpy.round(grid_point_heights_m_asl).astype(
int)
netcdf_dataset.close()
return (field_matrix, grid_point_heights_m_asl, grid_point_latitudes_deg,
grid_point_longitudes_deg)
def read_data_from_sparse_grid_file(netcdf_file_name, field_name_orig=None,
data_source=None, sentinel_values=None,
raise_error_if_fails=True):
"""Reads sparse radar grid from raw (either MYRORSS or MRMS) file.
This file should contain one radar field at one height and one time step.
:param netcdf_file_name: Path to input file.
:param field_name_orig: Name of radar field in original (either MYRORSS or
MRMS) format.
:param data_source: Data source (either "myrorss" or "mrms").
:param sentinel_values: 1-D numpy array of sentinel values.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error. If False and file cannot be
opened, this method will return None.
:return: sparse_grid_table: pandas DataFrame with the following columns.
Each row corresponds to one grid cell.
sparse_grid_table.grid_row: Row index.
sparse_grid_table.grid_column: Column index.
sparse_grid_table.<field_name>: Radar measurement (field_name is determined
by the method `field_name_orig_to_new`).
sparse_grid_table.num_grid_cells: Number of consecutive grid cells --
starting at the current one and counting along rows first, columns
second -- with the same radar measurement.
"""
error_checking.assert_file_exists(netcdf_file_name)
error_checking.assert_is_numpy_array_without_nan(sentinel_values)
error_checking.assert_is_numpy_array(sentinel_values, num_dimensions=1)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
field_name = _field_name_orig_to_new(field_name_orig,
data_source=data_source)
num_values = len(netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG])
if num_values == 0:
sparse_grid_dict = {
GRID_ROW_COLUMN: numpy.array([], dtype=int),
GRID_COLUMN_COLUMN: numpy.array([], dtype=int),
NUM_GRID_CELL_COLUMN: numpy.array([], dtype=int),
field_name: numpy.array([], dtype=int)}
else:
sparse_grid_dict = {
GRID_ROW_COLUMN: netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG][:],
GRID_COLUMN_COLUMN:
netcdf_dataset.variables[GRID_COLUMN_COLUMN_ORIG][:],
NUM_GRID_CELL_COLUMN:
netcdf_dataset.variables[NUM_GRID_CELL_COLUMN_ORIG][:],
field_name: netcdf_dataset.variables[field_name_orig][:]}
netcdf_dataset.close()
sparse_grid_table = pandas.DataFrame.from_dict(sparse_grid_dict)
return _remove_sentinels_from_sparse_grid(
sparse_grid_table, field_name, sentinel_values)
def read_classifications(netcdf_file_name):
"""Reads echo classifications from NetCDF file.
:param netcdf_file_name: Path to input file.
:return: convective_flag_matrix: M-by-N numpy array of Boolean flags (True
if convective, False if not).
:return: other_metadata_dict: Dictionary with the following keys.
other_metadata_dict['valid_time_unix_sec']: Valid time.
other_metadata_dict['grid_point_latitudes_deg']: length-M numpy array with
latitudes (deg N) of grid points.
other_metadata_dict['grid_point_longitudes_deg']: length-N numpy array with
longitudes (deg E) of grid points.
:return: option_dict: See doc for `find_convective_pixels`.
"""
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name)
option_dict = {
PEAKEDNESS_NEIGH_KEY:
getattr(netcdf_dataset, PEAKEDNESS_NEIGH_KEY),
MAX_PEAKEDNESS_HEIGHT_KEY:
getattr(netcdf_dataset, MAX_PEAKEDNESS_HEIGHT_KEY),
HALVE_RESOLUTION_KEY:
bool(getattr(netcdf_dataset, HALVE_RESOLUTION_KEY)),
MIN_ECHO_TOP_KEY:
getattr(netcdf_dataset, MIN_ECHO_TOP_KEY),
ECHO_TOP_LEVEL_KEY:
getattr(netcdf_dataset, ECHO_TOP_LEVEL_KEY),
MIN_COMPOSITE_REFL_CRITERION1_KEY:
getattr(netcdf_dataset, MIN_COMPOSITE_REFL_CRITERION1_KEY),
MIN_COMPOSITE_REFL_CRITERION5_KEY:
getattr(netcdf_dataset, MIN_COMPOSITE_REFL_CRITERION5_KEY),
MIN_COMPOSITE_REFL_AML_KEY:
getattr(netcdf_dataset, MIN_COMPOSITE_REFL_AML_KEY)
}
if option_dict[MIN_COMPOSITE_REFL_CRITERION1_KEY] < 0:
option_dict[MIN_COMPOSITE_REFL_CRITERION1_KEY] = None
option_dict = _check_input_args(option_dict)
other_metadata_dict = {
VALID_TIME_KEY:
int(getattr(netcdf_dataset, VALID_TIME_KEY)),
LATITUDES_KEY:
numpy.array(netcdf_dataset.variables[LATITUDES_KEY][:], dtype=float),
LONGITUDES_KEY:
numpy.array(netcdf_dataset.variables[LONGITUDES_KEY][:], dtype=float)
}
convective_flag_matrix = numpy.array(
netcdf_dataset.variables[FLAG_MATRIX_KEY][:], dtype=bool)
netcdf_dataset.close()
return convective_flag_matrix, other_metadata_dict, option_dict
def read_metadata_from_full_grid_file(netcdf_file_name,
raise_error_if_fails=True):
"""Reads metadata from full-grid (not sparse-grid) file.
This file should contain all radar variables for one time step.
:param netcdf_file_name: Path to input file.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error. If False and file cannot be
opened, will return None.
:return: metadata_dict: Dictionary with the following keys.
metadata_dict['nw_grid_point_lat_deg']: Latitude (deg N) of northwesternmost
grid point.
metadata_dict['nw_grid_point_lng_deg']: Longitude (deg E) of
northwesternmost grid point.
metadata_dict['lat_spacing_deg']: Spacing (deg N) between adjacent rows.
metadata_dict['lng_spacing_deg']: Spacing (deg E) between adjacent columns.
metadata_dict['num_lat_in_grid']: Number of rows (unique grid-point
latitudes).
metadata_dict['num_lng_in_grid']: Number of columns (unique grid-point
longitudes).
metadata_dict['unix_time_sec']: Valid time.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
grid_point_latitudes_deg = numpy.array(
netcdf_dataset.variables[LATITUDE_NAME_ORIG])
grid_point_longitudes_deg = lng_conversion.convert_lng_positive_in_west(
numpy.array(netcdf_dataset.variables[LONGITUDE_NAME_ORIG]))
metadata_dict = {
radar_utils.NW_GRID_POINT_LAT_COLUMN:
numpy.max(grid_point_latitudes_deg),
radar_utils.NW_GRID_POINT_LNG_COLUMN:
numpy.min(grid_point_longitudes_deg),
radar_utils.LAT_SPACING_COLUMN:
numpy.mean(numpy.diff(grid_point_latitudes_deg)),
radar_utils.LNG_SPACING_COLUMN:
numpy.mean(numpy.diff(grid_point_longitudes_deg)),
radar_utils.NUM_LAT_COLUMN:
len(grid_point_latitudes_deg),
radar_utils.NUM_LNG_COLUMN:
len(grid_point_longitudes_deg),
radar_utils.UNIX_TIME_COLUMN:
_time_from_gridrad_to_unix(netcdf_dataset.variables[TIME_NAME_ORIG][0])
}
netcdf_dataset.close()
return metadata_dict
def read_data_from_full_grid_file(netcdf_file_name, metadata_dict,
raise_error_if_fails=True):
"""Reads full radar grid from raw (either MYRORSS or MRMS) file.
This file should contain one radar field at one height and one time step.
M = number of rows (unique grid-point latitudes)
N = number of columns (unique grid-point longitudes)
:param netcdf_file_name: Path to input file.
:param metadata_dict: Dictionary with metadata for NetCDF file, created by
read_metadata_from_raw_file.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error. If False and file cannot be
opened, this method will return None for all output variables.
:return: field_matrix: M-by-N numpy array with values of radar field.
:return: unique_grid_point_lat_deg: length-M numpy array of grid-point
latitudes (deg N). If array is increasing (decreasing), latitude
increases (decreases) while traveling down the columns of field_matrix.
:return: unique_grid_point_lng_deg: length-N numpy array of grid-point
longitudes (deg E). If array is increasing (decreasing), longitude
increases (decreases) while traveling right across the rows of
field_matrix.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None, None, None
field_matrix = netcdf_dataset.variables[
metadata_dict[FIELD_NAME_COLUMN_ORIG]]
netcdf_dataset.close()
min_latitude_deg = metadata_dict[NW_GRID_POINT_LAT_COLUMN] - (
metadata_dict[LAT_SPACING_COLUMN] * (metadata_dict[NUM_LAT_COLUMN] - 1))
unique_grid_point_lat_deg, unique_grid_point_lng_deg = (
grids.get_latlng_grid_points(
min_latitude_deg=min_latitude_deg,
min_longitude_deg=metadata_dict[NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[LNG_SPACING_COLUMN],
num_rows=metadata_dict[NUM_LAT_COLUMN],
num_columns=metadata_dict[NUM_LNG_COLUMN]))
field_matrix = _remove_sentinels_from_full_grid(
field_matrix, metadata_dict[SENTINEL_VALUE_COLUMN])
return (numpy.flipud(field_matrix), unique_grid_point_lat_deg[::-1],
unique_grid_point_lng_deg)
def read_data_from_full_grid_file(netcdf_file_name,
metadata_dict,
raise_error_if_fails=True):
"""Reads full radar grid from raw (either MYRORSS or MRMS) file.
This file should contain one radar field at one height and valid time.
:param netcdf_file_name: Path to input file.
:param metadata_dict: Dictionary created by `read_metadata_from_raw_file`.
:param raise_error_if_fails: Boolean flag. If True and file cannot be read,
this method will raise an error. If False and file cannot be read, will
return None for all output vars.
:return: field_matrix: M-by-N numpy array with radar field. Latitude
increases while moving up each column, and longitude increases while
moving right along each row.
:return: grid_point_latitudes_deg: length-M numpy array of grid-point
latitudes (deg N). This array is monotonically decreasing.
:return: grid_point_longitudes_deg: length-N numpy array of grid-point
longitudes (deg E). This array is monotonically increasing.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None, None, None
field_matrix = netcdf_dataset.variables[
metadata_dict[FIELD_NAME_COLUMN_ORIG]]
netcdf_dataset.close()
min_latitude_deg = metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN] - (
metadata_dict[radar_utils.LAT_SPACING_COLUMN] *
(metadata_dict[radar_utils.NUM_LAT_COLUMN] - 1))
grid_point_latitudes_deg, grid_point_longitudes_deg = (
grids.get_latlng_grid_points(
min_latitude_deg=min_latitude_deg,
min_longitude_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN],
num_rows=metadata_dict[radar_utils.NUM_LAT_COLUMN],
num_columns=metadata_dict[radar_utils.NUM_LNG_COLUMN]))
field_matrix = _remove_sentinels_from_full_grid(
field_matrix, metadata_dict[radar_utils.SENTINEL_VALUE_COLUMN])
return (numpy.flipud(field_matrix), grid_point_latitudes_deg[::-1],
grid_point_longitudes_deg)
def read_features(netcdf_file_name):
"""Reads features (activations of intermediate layer) from NetCDF file.
:param netcdf_file_name: Path to input file.
:return: feature_matrix: E-by-Z numpy array of features.
:return: target_values: length-E numpy array of target values. All in
0...(K - 1), where K = number of classes.
:return: num_classes: Number of classes.
"""
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name=netcdf_file_name,
raise_error_if_fails=True)
feature_matrix = numpy.array(
netcdf_dataset.variables[FEATURE_MATRIX_KEY][:])
target_values = numpy.array(netcdf_dataset.variables[TARGET_VALUES_KEY][:],
dtype=int)
num_classes = getattr(netcdf_dataset, NUM_CLASSES_KEY)
netcdf_dataset.close()
return feature_matrix, target_values, num_classes
def read_target_values(netcdf_file_name, target_names=None):
"""Reads target values from NetCDF file.
E = number of examples (storm objects)
T = number of target variables
:param netcdf_file_name: Path to input file.
:param target_names: 1-D list with names of target variables to read. If
None, will read all target variables.
:return: storm_label_dict: Dictionary with the following keys.
storm_label_dict['full_id_strings']: length-E list of full storm IDs.
storm_label_dict['valid_times_unix_sec']: length-E numpy array of valid
times.
storm_label_dict['target_names']: length-T list with names of target
variables.
storm_label_dict['target_matrix']: E-by-T of target values (integer class
labels).
"""
netcdf_dataset = netcdf_io.open_netcdf(
netcdf_file_name=netcdf_file_name, raise_error_if_fails=True)
try:
full_id_strings = netCDF4.chartostring(
netcdf_dataset.variables[FULL_IDS_KEY][:]
)
except KeyError:
full_id_strings = netCDF4.chartostring(
netcdf_dataset.variables['storm_ids'][:]
)
valid_times_unix_sec = numpy.array(
netcdf_dataset.variables[VALID_TIMES_KEY][:], dtype=int
)
if target_names is None:
target_names = list(netcdf_dataset.variables.keys())
target_names.remove(FULL_IDS_KEY)
target_names.remove(VALID_TIMES_KEY)
error_checking.assert_is_string_list(target_names)
error_checking.assert_is_numpy_array(
numpy.array(target_names), num_dimensions=1
)
num_storm_objects = len(full_id_strings)
target_matrix = None
for this_target_name in target_names:
these_target_values = numpy.array(
netcdf_dataset.variables[this_target_name][:], dtype=int
)
these_target_values = numpy.reshape(
these_target_values, (num_storm_objects, 1)
)
if target_matrix is None:
target_matrix = these_target_values + 0
else:
target_matrix = numpy.concatenate(
(target_matrix, these_target_values), axis=1
)
netcdf_dataset.close()
return {
FULL_IDS_KEY: [str(f) for f in full_id_strings],
VALID_TIMES_KEY: valid_times_unix_sec,
TARGET_NAMES_KEY: target_names,
TARGET_MATRIX_KEY: target_matrix
}
def read_winds_from_raw_file(netcdf_file_name, secondary_source=None,
raise_error_if_fails=True):
"""Reads wind observations from raw file.
This file should contain all fields for one secondary data source and one
hour.
:param netcdf_file_name: Path to input file.
:param secondary_source: String ID for secondary data source.
:param raise_error_if_fails: Boolean flag. If True and the read fails, this
method will raise an error. If False and the read fails, this method
will return None.
:return: wind_table: If file cannot be opened and raise_error_if_fails =
False, this is None. Otherwise, it is a pandas DataFrame with the
following columns.
wind_table.station_id: String ID for station.
wind_table.station_name: Verbose name for station.
wind_table.latitude_deg: Latitude (deg N).
wind_table.longitude_deg: Longitude (deg E).
wind_table.elevation_m_asl: Elevation (metres above sea level).
wind_table.unix_time_sec: Observation time (seconds since 0000 UTC 1 Jan
1970).
wind_table.wind_speed_m_s01: Speed of sustained wind (m/s).
wind_table.wind_direction_deg: Direction of sustained wind (degrees of
origin -- i.e., direction that the wind is coming from -- as per
meteorological convention).
wind_table.wind_gust_speed_m_s01: Speed of wind gust (m/s).
wind_table.wind_gust_direction_deg: Direction of wind gust (degrees of
origin).
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
# TODO(thunderhoser): This is hacky (accounts for length-0 arrays of station
# names). Find a better way to handle this exception.
try:
station_names = _char_matrix_to_string_list(
netcdf_dataset.variables[STATION_NAME_COLUMN_ORIG][:])
except IndexError:
return None
try:
station_ids = _char_matrix_to_string_list(
netcdf_dataset.variables[STATION_ID_COLUMN_ORIG][:])
except KeyError:
station_ids = station_names
for i in range(len(station_ids)):
station_ids[i] = raw_wind_io.append_source_to_station_id(
station_ids[i], primary_source=raw_wind_io.MADIS_DATA_SOURCE,
secondary_source=secondary_source)
try:
unix_times_sec = netcdf_dataset.variables[TIME_COLUMN_ORIG][:]
except KeyError:
unix_times_sec = netcdf_dataset.variables[TIME_COLUMN_ORIG_BACKUP][:]
wind_speeds_m_s01 = netcdf_dataset.variables[WIND_SPEED_COLUMN_ORIG][:]
wind_speed_quality_flags = netcdf_dataset.variables[
WIND_SPEED_FLAG_COLUMN_ORIG][:]
num_observations = len(wind_speeds_m_s01)
try:
wind_directions_deg = netcdf_dataset.variables[WIND_DIR_COLUMN_ORIG][:]
wind_dir_quality_flags = netcdf_dataset.variables[
WIND_DIR_FLAG_COLUMN_ORIG][:]
except KeyError:
wind_directions_deg = numpy.full(num_observations, numpy.nan)
wind_dir_quality_flags = [DEFAULT_QUALITY_FLAG] * num_observations
try:
wind_gust_speeds_m_s01 = netcdf_dataset.variables[
WIND_GUST_SPEED_COLUMN_ORIG][:]
wind_gust_speed_quality_flags = netcdf_dataset.variables[
WIND_GUST_SPEED_FLAG_COLUMN_ORIG][:]
except KeyError:
wind_gust_speeds_m_s01 = numpy.full(num_observations, numpy.nan)
wind_gust_speed_quality_flags = (
[DEFAULT_QUALITY_FLAG] * num_observations)
try:
wind_gust_directions_deg = netcdf_dataset.variables[
WIND_GUST_DIR_COLUMN_ORIG][:]
wind_gust_dir_quality_flags = netcdf_dataset.variables[
WIND_GUST_DIR_FLAG_COLUMN_ORIG][:]
except KeyError:
wind_gust_directions_deg = numpy.full(num_observations, numpy.nan)
wind_gust_dir_quality_flags = [DEFAULT_QUALITY_FLAG] * num_observations
wind_dict = {raw_wind_io.STATION_ID_COLUMN: station_ids,
raw_wind_io.STATION_NAME_COLUMN: station_names,
raw_wind_io.LATITUDE_COLUMN: netcdf_dataset.variables[
LATITUDE_COLUMN_ORIG][:],
raw_wind_io.LONGITUDE_COLUMN: netcdf_dataset.variables[
LONGITUDE_COLUMN_ORIG][:],
raw_wind_io.ELEVATION_COLUMN: netcdf_dataset.variables[
ELEVATION_COLUMN_ORIG][:],
raw_wind_io.TIME_COLUMN: numpy.array(unix_times_sec).astype(
int),
raw_wind_io.WIND_SPEED_COLUMN: wind_speeds_m_s01,
raw_wind_io.WIND_DIR_COLUMN: wind_directions_deg,
raw_wind_io.WIND_GUST_SPEED_COLUMN: wind_gust_speeds_m_s01,
raw_wind_io.WIND_GUST_DIR_COLUMN: wind_gust_directions_deg,
WIND_SPEED_FLAG_COLUMN: wind_speed_quality_flags,
WIND_DIR_FLAG_COLUMN: wind_dir_quality_flags,
WIND_GUST_SPEED_FLAG_COLUMN: wind_gust_speed_quality_flags,
WIND_GUST_DIR_FLAG_COLUMN: wind_gust_dir_quality_flags}
netcdf_dataset.close()
wind_table = pandas.DataFrame.from_dict(wind_dict)
wind_table = _remove_invalid_wind_rows(wind_table)
return _remove_low_quality_data(wind_table)
def read_field_from_full_grid_file(netcdf_file_name,
field_name=None,
metadata_dict=None,
raise_error_if_fails=True):
"""Reads one radar field from full-grid (not sparse-grid) file.
This file should contain all radar variables for one time step.
M = number of rows (unique grid-point latitudes)
N = number of columns (unique grid-point longitudes)
H = number of height levels (unique grid-point heights)
:param netcdf_file_name: Path to input file.
:param field_name: Name of radar field.
:param metadata_dict: Dictionary created by
read_metadata_from_full_grid_file.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error. If False and file cannot be
opened, will return None for all output variables.
:return: field_matrix: H-by-M-by-N numpy array with values of radar field.
:return: unique_grid_point_heights_m_asl: length-H numpy array of grid-point
heights (metres above sea level). If array is increasing
(decreasing), height increases (decreases) with the first index of
field_matrix.
:return: unique_grid_point_lat_deg: length-M numpy array of grid-point
latitudes (deg N). If array is increasing (decreasing), latitude
increases (decreases) with the second index of field_matrix.
:return: unique_grid_point_lng_deg: length-N numpy array of grid-point
longitudes (deg E). If array is increasing (decreasing), longitude
increases (decreases) with the third index of field_matrix.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None, None, None, None
field_name_orig = _field_name_new_to_orig(field_name)
field_matrix = numpy.array(
netcdf_dataset.variables[field_name_orig][0, :, :, :])
min_latitude_deg = metadata_dict[radar_io.NW_GRID_POINT_LAT_COLUMN] - (
metadata_dict[radar_io.LAT_SPACING_COLUMN] *
(metadata_dict[radar_io.NUM_LAT_COLUMN] - 1))
unique_grid_point_lat_deg, unique_grid_point_lng_deg = (
grids.get_latlng_grid_points(
min_latitude_deg=min_latitude_deg,
min_longitude_deg=metadata_dict[radar_io.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_io.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_io.LNG_SPACING_COLUMN],
num_rows=metadata_dict[radar_io.NUM_LAT_COLUMN],
num_columns=metadata_dict[radar_io.NUM_LNG_COLUMN]))
max_height_m_asl = metadata_dict[MIN_GRID_POINT_HEIGHT_COLUMN] + (
metadata_dict[HEIGHT_SPACING_COLUMN] *
(metadata_dict[NUM_HEIGHTS_COLUMN] - 1))
unique_grid_point_heights_m_asl = numpy.linspace(
metadata_dict[MIN_GRID_POINT_HEIGHT_COLUMN],
max_height_m_asl,
num=metadata_dict[NUM_HEIGHTS_COLUMN])
netcdf_dataset.close()
return (field_matrix, unique_grid_point_heights_m_asl,
unique_grid_point_lat_deg, unique_grid_point_lng_deg)
def read_polygons_from_netcdf(netcdf_file_name,
metadata_dict=None,
spc_date_unix_sec=None,
tracking_start_time_unix_sec=None,
tracking_end_time_unix_sec=None,
raise_error_if_fails=True):
"""Reads storm polygons (outlines of storm cells) from NetCDF file.
P = number of grid points in storm cell (different for each storm cell)
V = number of vertices in storm polygon (different for each storm cell)
If file cannot be opened, returns None.
:param netcdf_file_name: Path to input file.
:param metadata_dict: Dictionary with metadata for NetCDF file, created by
`radar_io.read_metadata_from_raw_file`.
:param spc_date_unix_sec: SPC date;
:param tracking_start_time_unix_sec: Start time for tracking period. This
can be found by `get_start_end_times_for_spc_date`.
:param tracking_end_time_unix_sec: End time for tracking period. This can
be found by `get_start_end_times_for_spc_date`.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error.
:return: polygon_table: If file cannot be opened and raise_error_if_fails =
False, this is None. Otherwise, it is a pandas DataFrame with the
following columns.
polygon_table.storm_id: String ID for storm cell.
polygon_table.unix_time_sec: Time in Unix format.
polygon_table.spc_date_unix_sec: SPC date in Unix format.
polygon_table.tracking_start_time_unix_sec: Start time for tracking period.
polygon_table.tracking_end_time_unix_sec: End time for tracking period.
polygon_table.centroid_lat_deg: Latitude at centroid of storm cell (deg N).
polygon_table.centroid_lng_deg: Longitude at centroid of storm cell (deg E).
polygon_table.grid_point_latitudes_deg: length-P numpy array with latitudes
(deg N) of grid points in storm cell.
polygon_table.grid_point_longitudes_deg: length-P numpy array with
longitudes (deg E) of grid points in storm cell.
polygon_table.grid_point_rows: length-P numpy array with row indices (all
integers) of grid points in storm cell.
polygon_table.grid_point_columns: length-P numpy array with column indices
(all integers) of grid points in storm cell.
polygon_table.polygon_object_latlng: Instance of `shapely.geometry.Polygon`
with vertices in lat-long coordinates.
polygon_table.polygon_object_rowcol: Instance of `shapely.geometry.Polygon`
with vertices in row-column coordinates.
"""
error_checking.assert_file_exists(netcdf_file_name)
error_checking.assert_is_integer(spc_date_unix_sec)
error_checking.assert_is_not_nan(spc_date_unix_sec)
error_checking.assert_is_integer(tracking_start_time_unix_sec)
error_checking.assert_is_not_nan(tracking_start_time_unix_sec)
error_checking.assert_is_integer(tracking_end_time_unix_sec)
error_checking.assert_is_not_nan(tracking_end_time_unix_sec)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
storm_id_var_name = metadata_dict[radar_io.FIELD_NAME_COLUMN]
storm_id_var_name_orig = metadata_dict[radar_io.FIELD_NAME_COLUMN_ORIG]
num_values = len(netcdf_dataset.variables[radar_io.GRID_ROW_COLUMN_ORIG])
if num_values == 0:
sparse_grid_dict = {
radar_io.GRID_ROW_COLUMN: numpy.array([], dtype=int),
radar_io.GRID_COLUMN_COLUMN: numpy.array([], dtype=int),
radar_io.NUM_GRID_CELL_COLUMN: numpy.array([], dtype=int),
storm_id_var_name: numpy.array([], dtype=int)
}
else:
sparse_grid_dict = {
radar_io.GRID_ROW_COLUMN:
netcdf_dataset.variables[radar_io.GRID_ROW_COLUMN_ORIG][:],
radar_io.GRID_COLUMN_COLUMN:
netcdf_dataset.variables[radar_io.GRID_COLUMN_COLUMN_ORIG][:],
radar_io.NUM_GRID_CELL_COLUMN:
netcdf_dataset.variables[radar_io.NUM_GRID_CELL_COLUMN_ORIG][:],
storm_id_var_name:
netcdf_dataset.variables[storm_id_var_name_orig][:]
}
netcdf_dataset.close()
sparse_grid_table = pandas.DataFrame.from_dict(sparse_grid_dict)
numeric_storm_id_matrix, _, _ = (radar_s2f.sparse_to_full_grid(
sparse_grid_table, metadata_dict))
polygon_table = _storm_id_matrix_to_coord_lists(numeric_storm_id_matrix)
num_storms = len(polygon_table.index)
unix_times_sec = numpy.full(num_storms,
metadata_dict[radar_io.UNIX_TIME_COLUMN],
dtype=int)
spc_dates_unix_sec = numpy.full(num_storms, spc_date_unix_sec, dtype=int)
tracking_start_times_unix_sec = numpy.full(num_storms,
tracking_start_time_unix_sec,
dtype=int)
tracking_end_times_unix_sec = numpy.full(num_storms,
tracking_end_time_unix_sec,
dtype=int)
spc_date_string = time_conversion.time_to_spc_date_string(
spc_date_unix_sec)
storm_ids = _append_spc_date_to_storm_ids(
polygon_table[tracking_io.STORM_ID_COLUMN].values, spc_date_string)
simple_array = numpy.full(num_storms, numpy.nan)
object_array = numpy.full(num_storms, numpy.nan, dtype=object)
nested_array = polygon_table[[
tracking_io.STORM_ID_COLUMN, tracking_io.STORM_ID_COLUMN
]].values.tolist()
argument_dict = {
tracking_io.STORM_ID_COLUMN: storm_ids,
tracking_io.TIME_COLUMN: unix_times_sec,
tracking_io.SPC_DATE_COLUMN: spc_dates_unix_sec,
tracking_io.TRACKING_START_TIME_COLUMN: tracking_start_times_unix_sec,
tracking_io.TRACKING_END_TIME_COLUMN: tracking_end_times_unix_sec,
tracking_io.CENTROID_LAT_COLUMN: simple_array,
tracking_io.CENTROID_LNG_COLUMN: simple_array,
tracking_io.GRID_POINT_LAT_COLUMN: nested_array,
tracking_io.GRID_POINT_LNG_COLUMN: nested_array,
tracking_io.POLYGON_OBJECT_LATLNG_COLUMN: object_array,
tracking_io.POLYGON_OBJECT_ROWCOL_COLUMN: object_array
}
polygon_table = polygon_table.assign(**argument_dict)
for i in range(num_storms):
these_vertex_rows, these_vertex_columns = (
polygons.grid_points_in_poly_to_vertices(
polygon_table[tracking_io.GRID_POINT_ROW_COLUMN].values[i],
polygon_table[tracking_io.GRID_POINT_COLUMN_COLUMN].values[i]))
(polygon_table[tracking_io.GRID_POINT_ROW_COLUMN].values[i],
polygon_table[tracking_io.GRID_POINT_COLUMN_COLUMN].values[i]) = (
polygons.simple_polygon_to_grid_points(these_vertex_rows,
these_vertex_columns))
(polygon_table[tracking_io.GRID_POINT_LAT_COLUMN].values[i],
polygon_table[tracking_io.GRID_POINT_LNG_COLUMN].values[i]) = (
radar_io.rowcol_to_latlng(
polygon_table[tracking_io.GRID_POINT_ROW_COLUMN].values[i],
polygon_table[tracking_io.GRID_POINT_COLUMN_COLUMN].values[i],
nw_grid_point_lat_deg=metadata_dict[
radar_io.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_io.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_io.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_io.LNG_SPACING_COLUMN]))
these_vertex_lat_deg, these_vertex_lng_deg = radar_io.rowcol_to_latlng(
these_vertex_rows,
these_vertex_columns,
nw_grid_point_lat_deg=metadata_dict[
radar_io.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_io.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_io.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_io.LNG_SPACING_COLUMN])
(polygon_table[tracking_io.CENTROID_LAT_COLUMN].values[i],
polygon_table[tracking_io.CENTROID_LNG_COLUMN].values[i]) = (
polygons.get_latlng_centroid(these_vertex_lat_deg,
these_vertex_lng_deg))
polygon_table[tracking_io.POLYGON_OBJECT_ROWCOL_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(these_vertex_columns,
these_vertex_rows))
polygon_table[tracking_io.POLYGON_OBJECT_LATLNG_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(these_vertex_lng_deg,
these_vertex_lat_deg))
return polygon_table
def read_data_from_sparse_grid_file(netcdf_file_name,
field_name_orig,
data_source,
sentinel_values,
raise_error_if_fails=True):
"""Reads sparse radar grid from raw (either MYRORSS or MRMS) file.
This file should contain one radar field at one height and valid time.
:param netcdf_file_name: Path to input file.
:param field_name_orig: Name of radar field in original (either MYRORSS or
MRMS) format.
:param data_source: Data source (string).
:param sentinel_values: 1-D numpy array of sentinel values.
:param raise_error_if_fails: Boolean flag. If True and file cannot be read,
this method will raise an error. If False and file cannot be read, will
return None.
:return: sparse_grid_table: pandas DataFrame with the following columns.
Each row corresponds to one grid point.
sparse_grid_table.grid_row: Row index.
sparse_grid_table.grid_column: Column index.
sparse_grid_table.<field_name>: Radar measurement (column name is produced
by _field_name_orig_to_new).
sparse_grid_table.num_grid_cells: Number of consecutive grid points with the
same radar measurement. Counting is row-major (to the right along the
row, then down to the next column if necessary).
"""
error_checking.assert_file_exists(netcdf_file_name)
error_checking.assert_is_numpy_array_without_nan(sentinel_values)
error_checking.assert_is_numpy_array(sentinel_values, num_dimensions=1)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
field_name = radar_utils.field_name_orig_to_new(
field_name_orig=field_name_orig, data_source_name=data_source)
num_values = len(netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG])
if num_values == 0:
sparse_grid_dict = {
GRID_ROW_COLUMN: numpy.array([], dtype=int),
GRID_COLUMN_COLUMN: numpy.array([], dtype=int),
NUM_GRID_CELL_COLUMN: numpy.array([], dtype=int),
field_name: numpy.array([])
}
else:
sparse_grid_dict = {
GRID_ROW_COLUMN:
netcdf_dataset.variables[GRID_ROW_COLUMN_ORIG][:],
GRID_COLUMN_COLUMN:
netcdf_dataset.variables[GRID_COLUMN_COLUMN_ORIG][:],
NUM_GRID_CELL_COLUMN:
netcdf_dataset.variables[NUM_GRID_CELL_COLUMN_ORIG][:],
field_name:
netcdf_dataset.variables[field_name_orig][:]
}
netcdf_dataset.close()
sparse_grid_table = pandas.DataFrame.from_dict(sparse_grid_dict)
return _remove_sentinels_from_sparse_grid(sparse_grid_table,
field_name=field_name,
sentinel_values=sentinel_values)
def read_metadata_from_raw_file(netcdf_file_name, data_source=None,
raise_error_if_fails=True):
"""Reads metadata raw (either MYRORSS or MRMS) file..
This file should contain one radar field at one height and one time step.
:param netcdf_file_name: Path to input file.
:param data_source: Data source (either "myrorss" or "mrms").
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error. If False and file cannot be
opened, this method will return None.
:return: metadata_dict: Dictionary with the following keys.
metadata_dict['nw_grid_point_lat_deg']: Latitude (deg N) of northwesternmost
grid point.
metadata_dict['nw_grid_point_lng_deg']: Longitude (deg E) of
northwesternmost grid point.
metadata_dict['lat_spacing_deg']: Spacing (deg N) between adjacent rows.
metadata_dict['lng_spacing_deg']: Spacing (deg E) between adjacent columns.
metadata_dict['num_lat_in_grid']: Number of rows (unique grid-point
latitudes).
metadata_dict['num_lng_in_grid']: Number of columns (unique grid-point
longitudes).
metadata_dict['height_m_agl']: Height (metres above ground level).
metadata_dict['unix_time_sec']: Time in Unix format.
metadata_dict['field_name']: Name of radar field in new format.
metadata_dict['field_name_orig']: Name of radar field in original (MYRORSS
or MRMS) format.
metadata_dict['sentinel_values']: 1-D numpy array of sentinel values.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
field_name_orig = str(getattr(netcdf_dataset, FIELD_NAME_COLUMN_ORIG))
metadata_dict = {
NW_GRID_POINT_LAT_COLUMN: getattr(netcdf_dataset,
NW_GRID_POINT_LAT_COLUMN_ORIG),
NW_GRID_POINT_LNG_COLUMN: lng_conversion.convert_lng_positive_in_west(
getattr(netcdf_dataset, NW_GRID_POINT_LNG_COLUMN_ORIG),
allow_nan=False),
LAT_SPACING_COLUMN: getattr(netcdf_dataset, LAT_SPACING_COLUMN_ORIG),
LNG_SPACING_COLUMN: getattr(netcdf_dataset, LNG_SPACING_COLUMN_ORIG),
NUM_LAT_COLUMN: netcdf_dataset.dimensions[NUM_LAT_COLUMN_ORIG].size + 1,
NUM_LNG_COLUMN: netcdf_dataset.dimensions[NUM_LNG_COLUMN_ORIG].size + 1,
HEIGHT_COLUMN: getattr(netcdf_dataset, HEIGHT_COLUMN_ORIG),
UNIX_TIME_COLUMN: getattr(netcdf_dataset, UNIX_TIME_COLUMN_ORIG),
FIELD_NAME_COLUMN_ORIG: field_name_orig,
FIELD_NAME_COLUMN: _field_name_orig_to_new(field_name_orig,
data_source=data_source)}
metadata_dict[NW_GRID_POINT_LAT_COLUMN] = rounder.floor_to_nearest(
metadata_dict[NW_GRID_POINT_LAT_COLUMN],
metadata_dict[LAT_SPACING_COLUMN])
metadata_dict[NW_GRID_POINT_LNG_COLUMN] = rounder.ceiling_to_nearest(
metadata_dict[NW_GRID_POINT_LNG_COLUMN],
metadata_dict[LNG_SPACING_COLUMN])
sentinel_values = numpy.full(len(SENTINEL_VALUE_COLUMNS_ORIG), numpy.nan)
for i in range(len(SENTINEL_VALUE_COLUMNS_ORIG)):
sentinel_values[i] = getattr(netcdf_dataset,
SENTINEL_VALUE_COLUMNS_ORIG[i])
metadata_dict.update({SENTINEL_VALUE_COLUMN: sentinel_values})
netcdf_dataset.close()
return metadata_dict
def read_file(netcdf_file_name,
field_name,
valid_time_unix_sec,
pressure_level_mb=None):
"""Reads data from NetCDF file.
This method will extract one field at one pressure level (or surface) at one
time.
M = number of rows in grid
N = number of columns in grid
:param netcdf_file_name: Path to input file.
:param field_name: Field to extract (must be accepted by
`processed_narr_io.check_field_name`).
:param valid_time_unix_sec: Valid time.
:param pressure_level_mb: [used only if file contains isobaric data]
Pressure level to extract (millibars).
:return: data_matrix: M-by-N numpy array with values of the given field at
the given pressure level (or surface).
"""
field_name_orig = _std_to_netcdf_field_name(field_name)
valid_time_narr_hours = _unix_to_narr_time(valid_time_unix_sec)
if pressure_level_mb is None:
# TODO(thunderhoser): This is a HACK.
if field_name_orig == HEIGHT_NAME_NETCDF:
field_name_orig = 'pres'
if field_name_orig == VERTICAL_VELOCITY_NAME_NETCDF:
field_name_orig = 'vvel'
else:
error_checking.assert_is_integer(pressure_level_mb)
dataset_object = netcdf_io.open_netcdf(netcdf_file_name=netcdf_file_name,
raise_error_if_fails=True)
is_surface = PRESSURE_KEY not in dataset_object.variables
all_times_narr_hours = numpy.round(
dataset_object.variables[TIME_KEY]).astype(int)
time_index = numpy.where(
all_times_narr_hours == valid_time_narr_hours)[0][0]
if is_surface:
field_matrix = numpy.array(
dataset_object.variables[field_name_orig][time_index, ...])
else:
all_pressure_levels_mb = numpy.round(
dataset_object.variables[PRESSURE_KEY]).astype(int)
pressure_index = numpy.where(
all_pressure_levels_mb == pressure_level_mb)[0][0]
field_matrix = numpy.array(
dataset_object.variables[field_name_orig][time_index,
pressure_index, ...])
return _remove_sentinel_values(field_matrix)
def read_metadata_from_raw_file(netcdf_file_name,
data_source,
raise_error_if_fails=True):
"""Reads metadata from raw (either MYRORSS or MRMS) file.
This file should contain one radar field at one height and valid time.
:param netcdf_file_name: Path to input file.
:param data_source: Data source (string).
:param raise_error_if_fails: Boolean flag. If True and file cannot be read,
this method will raise an error. If False and file cannot be read, will
return None.
:return: metadata_dict: Dictionary with the following keys.
metadata_dict['nw_grid_point_lat_deg']: Latitude (deg N) of northwesternmost
grid point.
metadata_dict['nw_grid_point_lng_deg']: Longitude (deg E) of
northwesternmost grid point.
metadata_dict['lat_spacing_deg']: Spacing (deg N) between meridionally
adjacent grid points.
metadata_dict['lng_spacing_deg']: Spacing (deg E) between zonally adjacent
grid points.
metadata_dict['num_lat_in_grid']: Number of rows (unique grid-point
latitudes).
metadata_dict['num_lng_in_grid']: Number of columns (unique grid-point
longitudes).
metadata_dict['height_m_asl']: Radar height (metres above ground level).
metadata_dict['unix_time_sec']: Valid time.
metadata_dict['field_name']: Name of radar field in GewitterGefahr format.
metadata_dict['field_name_orig']: Name of radar field in original (either
MYRORSS or MRMS) format.
metadata_dict['sentinel_values']: 1-D numpy array of sentinel values.
"""
error_checking.assert_file_exists(netcdf_file_name)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
field_name_orig = str(getattr(netcdf_dataset, FIELD_NAME_COLUMN_ORIG))
metadata_dict = {
radar_utils.NW_GRID_POINT_LAT_COLUMN:
getattr(netcdf_dataset, NW_GRID_POINT_LAT_COLUMN_ORIG),
radar_utils.NW_GRID_POINT_LNG_COLUMN:
lng_conversion.convert_lng_positive_in_west(getattr(
netcdf_dataset, NW_GRID_POINT_LNG_COLUMN_ORIG),
allow_nan=False),
radar_utils.LAT_SPACING_COLUMN:
getattr(netcdf_dataset, LAT_SPACING_COLUMN_ORIG),
radar_utils.LNG_SPACING_COLUMN:
getattr(netcdf_dataset, LNG_SPACING_COLUMN_ORIG),
radar_utils.NUM_LAT_COLUMN:
netcdf_dataset.dimensions[NUM_LAT_COLUMN_ORIG].size + 1,
radar_utils.NUM_LNG_COLUMN:
netcdf_dataset.dimensions[NUM_LNG_COLUMN_ORIG].size + 1,
radar_utils.HEIGHT_COLUMN:
getattr(netcdf_dataset, HEIGHT_COLUMN_ORIG),
radar_utils.UNIX_TIME_COLUMN:
getattr(netcdf_dataset, UNIX_TIME_COLUMN_ORIG),
FIELD_NAME_COLUMN_ORIG:
field_name_orig,
radar_utils.FIELD_NAME_COLUMN:
radar_utils.field_name_orig_to_new(field_name_orig=field_name_orig,
data_source_name=data_source)
}
latitude_spacing_deg = metadata_dict[radar_utils.LAT_SPACING_COLUMN]
longitude_spacing_deg = metadata_dict[radar_utils.LNG_SPACING_COLUMN]
# TODO(thunderhoser): The following "if" condition is a hack. The purpose
# is to change grid corners only for actual MYRORSS data, not GridRad data
# in MYRORSS format.
if latitude_spacing_deg < 0.011 and longitude_spacing_deg < 0.011:
metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN] = (
rounder.floor_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LAT_COLUMN],
metadata_dict[radar_utils.LAT_SPACING_COLUMN]))
metadata_dict[radar_utils.NW_GRID_POINT_LNG_COLUMN] = (
rounder.ceiling_to_nearest(
metadata_dict[radar_utils.NW_GRID_POINT_LNG_COLUMN],
metadata_dict[radar_utils.LNG_SPACING_COLUMN]))
sentinel_values = []
for this_column in SENTINEL_VALUE_COLUMNS_ORIG:
sentinel_values.append(getattr(netcdf_dataset, this_column))
metadata_dict.update(
{radar_utils.SENTINEL_VALUE_COLUMN: numpy.array(sentinel_values)})
netcdf_dataset.close()
return metadata_dict
def read_polygons_from_netcdf(netcdf_file_name,
metadata_dict,
spc_date_string,
tracking_start_time_unix_sec,
tracking_end_time_unix_sec,
raise_error_if_fails=True):
"""Reads storm polygons (outlines of storm cells) from NetCDF file.
P = number of grid points in storm cell (different for each storm cell)
V = number of vertices in storm polygon (different for each storm cell)
If file cannot be opened, returns None.
:param netcdf_file_name: Path to input file.
:param metadata_dict: Dictionary with metadata for NetCDF file, created by
`myrorss_and_mrms_io.read_metadata_from_raw_file`.
:param spc_date_string: SPC date (format "yyyymmdd").
:param tracking_start_time_unix_sec: Start time for tracking period. This
can be found by `get_start_end_times_for_spc_date`.
:param tracking_end_time_unix_sec: End time for tracking period. This can
be found by `get_start_end_times_for_spc_date`.
:param raise_error_if_fails: Boolean flag. If True and file cannot be
opened, this method will raise an error.
:return: polygon_table: pandas DataFrame with the following columns. Each
row is one storm object.
polygon_table.primary_id_string: See documentation for
`storm_tracking_io.write_file`.
polygon_table.valid_time_unix_sec: Same.
polygon_table.spc_date_string: Same.
polygon_table.tracking_start_time_unix_sec: Same.
polygon_table.tracking_end_time_unix_sec: Same.
polygon_table.centroid_latitude_deg: Same.
polygon_table.centroid_longitude_deg: Same.
polygon_table.grid_point_latitudes_deg: Same.
polygon_table.grid_point_longitudes_deg: Same.
polygon_table.grid_point_rows: Same.
polygon_table.grid_point_columns: Same.
polygon_table.polygon_object_latlng_deg: Same.
polygon_table.polygon_object_rowcol: Same.
"""
error_checking.assert_file_exists(netcdf_file_name)
error_checking.assert_is_integer(tracking_start_time_unix_sec)
error_checking.assert_is_not_nan(tracking_start_time_unix_sec)
error_checking.assert_is_integer(tracking_end_time_unix_sec)
error_checking.assert_is_not_nan(tracking_end_time_unix_sec)
netcdf_dataset = netcdf_io.open_netcdf(netcdf_file_name,
raise_error_if_fails)
if netcdf_dataset is None:
return None
storm_id_column = metadata_dict[radar_utils.FIELD_NAME_COLUMN]
storm_id_column_orig = metadata_dict[
myrorss_and_mrms_io.FIELD_NAME_COLUMN_ORIG]
num_values = len(
netcdf_dataset.variables[myrorss_and_mrms_io.GRID_ROW_COLUMN_ORIG])
if num_values == 0:
sparse_grid_dict = {
myrorss_and_mrms_io.GRID_ROW_COLUMN: numpy.array([], dtype=int),
myrorss_and_mrms_io.GRID_COLUMN_COLUMN: numpy.array([], dtype=int),
myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN: numpy.array([],
dtype=int),
storm_id_column: numpy.array([], dtype=int)
}
else:
sparse_grid_dict = {
myrorss_and_mrms_io.GRID_ROW_COLUMN:
netcdf_dataset.variables[myrorss_and_mrms_io.GRID_ROW_COLUMN_ORIG]
[:],
myrorss_and_mrms_io.GRID_COLUMN_COLUMN:
netcdf_dataset.variables[
myrorss_and_mrms_io.GRID_COLUMN_COLUMN_ORIG][:],
myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN:
netcdf_dataset.variables[
myrorss_and_mrms_io.NUM_GRID_CELL_COLUMN_ORIG][:],
storm_id_column:
netcdf_dataset.variables[storm_id_column_orig][:]
}
netcdf_dataset.close()
sparse_grid_table = pandas.DataFrame.from_dict(sparse_grid_dict)
numeric_id_matrix = radar_s2f.sparse_to_full_grid(sparse_grid_table,
metadata_dict)[0]
polygon_table = _id_matrix_to_coord_lists(numeric_id_matrix)
num_storms = len(polygon_table.index)
valid_times_unix_sec = numpy.full(
num_storms, metadata_dict[radar_utils.UNIX_TIME_COLUMN], dtype=int)
spc_date_strings = num_storms * [
time_conversion.time_to_spc_date_string(valid_times_unix_sec[0])
]
tracking_start_times_unix_sec = numpy.full(num_storms,
tracking_start_time_unix_sec,
dtype=int)
tracking_end_times_unix_sec = numpy.full(num_storms,
tracking_end_time_unix_sec,
dtype=int)
simple_array = numpy.full(num_storms, numpy.nan)
object_array = numpy.full(num_storms, numpy.nan, dtype=object)
nested_array = polygon_table[[
tracking_utils.PRIMARY_ID_COLUMN, tracking_utils.PRIMARY_ID_COLUMN
]].values.tolist()
argument_dict = {
tracking_utils.VALID_TIME_COLUMN: valid_times_unix_sec,
tracking_utils.SPC_DATE_COLUMN: spc_date_strings,
tracking_utils.TRACKING_START_TIME_COLUMN:
tracking_start_times_unix_sec,
tracking_utils.TRACKING_END_TIME_COLUMN: tracking_end_times_unix_sec,
tracking_utils.CENTROID_LATITUDE_COLUMN: simple_array,
tracking_utils.CENTROID_LONGITUDE_COLUMN: simple_array,
tracking_utils.LATITUDES_IN_STORM_COLUMN: nested_array,
tracking_utils.LONGITUDES_IN_STORM_COLUMN: nested_array,
tracking_utils.LATLNG_POLYGON_COLUMN: object_array,
tracking_utils.ROWCOL_POLYGON_COLUMN: object_array
}
polygon_table = polygon_table.assign(**argument_dict)
for i in range(num_storms):
these_vertex_rows, these_vertex_columns = (
polygons.grid_points_in_poly_to_vertices(
grid_point_row_indices=polygon_table[
tracking_utils.ROWS_IN_STORM_COLUMN].values[i],
grid_point_column_indices=polygon_table[
tracking_utils.COLUMNS_IN_STORM_COLUMN].values[i]))
(polygon_table[tracking_utils.ROWS_IN_STORM_COLUMN].values[i],
polygon_table[tracking_utils.COLUMNS_IN_STORM_COLUMN].values[i]
) = polygons.simple_polygon_to_grid_points(
vertex_row_indices=these_vertex_rows,
vertex_column_indices=these_vertex_columns)
(polygon_table[tracking_utils.LATITUDES_IN_STORM_COLUMN].values[i],
polygon_table[tracking_utils.LONGITUDES_IN_STORM_COLUMN].values[i]
) = radar_utils.rowcol_to_latlng(
grid_rows=polygon_table[
tracking_utils.ROWS_IN_STORM_COLUMN].values[i],
grid_columns=polygon_table[
tracking_utils.COLUMNS_IN_STORM_COLUMN].values[i],
nw_grid_point_lat_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN])
these_vertex_lat_deg, these_vertex_lng_deg = (
radar_utils.rowcol_to_latlng(
grid_rows=these_vertex_rows,
grid_columns=these_vertex_columns,
nw_grid_point_lat_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LAT_COLUMN],
nw_grid_point_lng_deg=metadata_dict[
radar_utils.NW_GRID_POINT_LNG_COLUMN],
lat_spacing_deg=metadata_dict[radar_utils.LAT_SPACING_COLUMN],
lng_spacing_deg=metadata_dict[radar_utils.LNG_SPACING_COLUMN]))
(polygon_table[tracking_utils.CENTROID_LATITUDE_COLUMN].values[i],
polygon_table[tracking_utils.CENTROID_LONGITUDE_COLUMN].values[i]
) = geodetic_utils.get_latlng_centroid(
latitudes_deg=these_vertex_lat_deg,
longitudes_deg=these_vertex_lng_deg)
polygon_table[tracking_utils.ROWCOL_POLYGON_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(
exterior_x_coords=these_vertex_columns,
exterior_y_coords=these_vertex_rows))
polygon_table[tracking_utils.LATLNG_POLYGON_COLUMN].values[i] = (
polygons.vertex_arrays_to_polygon_object(
exterior_x_coords=these_vertex_lng_deg,
exterior_y_coords=these_vertex_lat_deg))
primary_id_strings = _append_spc_date_to_storm_ids(
primary_id_strings=polygon_table[
tracking_utils.PRIMARY_ID_COLUMN].values,
spc_date_string=spc_date_string)
return polygon_table.assign(
**{tracking_utils.PRIMARY_ID_COLUMN: primary_id_strings})
