def test_fluxes_increments_to_actual(self):
"""Ensures correct output from fluxes_increments_to_actual."""
this_example_dict = example_utils.fluxes_increments_to_actual(
copy.deepcopy(EXAMPLE_DICT_WITH_INCREMENTS))
self.assertTrue(
_compare_example_dicts(this_example_dict,
EXAMPLE_DICT_WITH_INCREMENTS))
def _get_predicted_heating_rates(prediction_example_dict,
pressure_matrix_pascals, model_metadata_dict):
"""Computes predicted heating rates from predicted flux-increment profiles.
:param prediction_example_dict: Dictionary with predictions. For a list of
keys, see doc for `example_io.read_file`.
:param pressure_matrix_pascals: See doc for `_get_unnormalized_pressure`.
:param model_metadata_dict: Same.
:return: prediction_example_dict: Same but with heating rates.
"""
num_examples = pressure_matrix_pascals.shape[0]
generator_option_dict = copy.deepcopy(
model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY])
this_dict = {
example_utils.VECTOR_PREDICTOR_NAMES_KEY:
[example_utils.PRESSURE_NAME],
example_utils.VECTOR_PREDICTOR_VALS_KEY:
numpy.expand_dims(pressure_matrix_pascals, axis=-1),
example_utils.SCALAR_PREDICTOR_NAMES_KEY: [],
example_utils.SCALAR_PREDICTOR_VALS_KEY:
numpy.full((num_examples, 0), 0.),
example_utils.VALID_TIMES_KEY:
numpy.full(num_examples, 0, dtype=int)
}
prediction_example_dict.update(this_dict)
prediction_example_dict = (
example_utils.fluxes_increments_to_actual(prediction_example_dict))
prediction_example_dict = example_utils.fluxes_to_heating_rate(
prediction_example_dict)
target_names = (generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY] +
generator_option_dict[neural_net.SCALAR_TARGET_NAMES_KEY])
return example_utils.subset_by_field(example_dict=prediction_example_dict,
field_names=target_names)
def read_file(netcdf_file_name, allow_bad_values=False):
"""Reads RRTM data from NetCDF file.
E = number of examples
H = number of heights
P_s = number of scalar predictors
P_v = number of vector predictors
T_s = number of scalar targets
T_v = number of vector targets
:param netcdf_file_name: Path to NetCDF file with learning examples.
:param allow_bad_values: Boolean flag. If True, will allow bad values and
remove examples that have bad values.
:return: example_dict: Dictionary with the following keys.
example_dict['scalar_predictor_matrix']: numpy array (E x P_s) with values
of scalar predictors.
example_dict['scalar_predictor_names']: list (length P_s) with names of
scalar predictors.
example_dict['vector_predictor_matrix']: numpy array (E x H x P_v) with
values of vector predictors.
example_dict['vector_predictor_names']: list (length P_v) with names of
vector predictors.
example_dict['scalar_target_matrix']: numpy array (E x T_s) with values of
scalar targets.
example_dict['scalar_target_names']: list (length T_s) with names of scalar
targets.
example_dict['vector_target_matrix']: numpy array (E x H x T_v) with values
of vector targets.
example_dict['vector_target_names']: list (length T_v) with names of vector
targets.
example_dict['valid_times_unix_sec']: length-E numpy array of valid times
(Unix seconds).
example_dict['heights_m_agl']: length-H numpy array of heights (metres above
ground level).
example_dict['standard_atmo_flags']: length-E numpy array of flags (each in
the list `STANDARD_ATMO_ENUMS`).
example_dict['example_id_strings']: length-E list of example IDs.
"""
error_checking.assert_is_boolean(allow_bad_values)
# TODO(thunderhoser): This is a HACK.
if not os.path.isfile(netcdf_file_name):
netcdf_file_name = netcdf_file_name.replace('/home/ryan.lagerquist',
'/home/ralager')
dataset_object = netCDF4.Dataset(netcdf_file_name)
example_dict = {
example_utils.SCALAR_PREDICTOR_NAMES_KEY:
copy.deepcopy(DEFAULT_SCALAR_PREDICTOR_NAMES),
example_utils.VECTOR_PREDICTOR_NAMES_KEY:
copy.deepcopy(DEFAULT_VECTOR_PREDICTOR_NAMES),
example_utils.SCALAR_TARGET_NAMES_KEY:
copy.deepcopy(DEFAULT_SCALAR_TARGET_NAMES),
example_utils.VECTOR_TARGET_NAMES_KEY:
copy.deepcopy(DEFAULT_VECTOR_TARGET_NAMES),
example_utils.VALID_TIMES_KEY:
numpy.array(dataset_object.variables[VALID_TIMES_KEY][:], dtype=int),
example_utils.HEIGHTS_KEY:
KM_TO_METRES *
numpy.array(dataset_object.variables[HEIGHTS_KEY][:], dtype=float),
example_utils.STANDARD_ATMO_FLAGS_KEY:
numpy.array(numpy.round(
dataset_object.variables[STANDARD_ATMO_FLAGS_KEY][:]),
dtype=int)
}
num_examples = len(example_dict[example_utils.VALID_TIMES_KEY])
num_heights = len(example_dict[example_utils.HEIGHTS_KEY])
num_scalar_predictors = len(DEFAULT_SCALAR_PREDICTOR_NAMES)
num_vector_predictors = len(DEFAULT_VECTOR_PREDICTOR_NAMES)
num_scalar_targets = len(DEFAULT_SCALAR_TARGET_NAMES)
num_vector_targets = len(DEFAULT_VECTOR_TARGET_NAMES)
scalar_predictor_matrix = numpy.full((num_examples, num_scalar_predictors),
numpy.nan)
vector_predictor_matrix = numpy.full(
(num_examples, num_heights, num_vector_predictors), numpy.nan)
scalar_target_matrix = numpy.full((num_examples, num_scalar_targets),
numpy.nan)
vector_target_matrix = numpy.full(
(num_examples, num_heights, num_vector_targets), numpy.nan)
for k in range(num_scalar_predictors):
this_predictor_name_orig = (
PREDICTOR_NAME_TO_ORIG[DEFAULT_SCALAR_PREDICTOR_NAMES[k]])
this_conversion_factor = (
PREDICTOR_NAME_TO_CONV_FACTOR[DEFAULT_SCALAR_PREDICTOR_NAMES[k]])
scalar_predictor_matrix[:, k] = this_conversion_factor * numpy.array(
dataset_object.variables[this_predictor_name_orig][:], dtype=float)
for k in range(num_vector_predictors):
this_predictor_name_orig = (
PREDICTOR_NAME_TO_ORIG[DEFAULT_VECTOR_PREDICTOR_NAMES[k]])
this_conversion_factor = (
PREDICTOR_NAME_TO_CONV_FACTOR[DEFAULT_VECTOR_PREDICTOR_NAMES[k]])
vector_predictor_matrix[..., k] = this_conversion_factor * numpy.array(
dataset_object.variables[this_predictor_name_orig][:], dtype=float)
if DEFAULT_VECTOR_PREDICTOR_NAMES[k] in [
example_utils.LIQUID_WATER_CONTENT_NAME,
example_utils.ICE_WATER_CONTENT_NAME
]:
vector_predictor_matrix[..., k] = _layerwise_water_path_to_content(
layerwise_path_matrix_kg_m02=vector_predictor_matrix[..., k],
heights_m_agl=example_dict[example_utils.HEIGHTS_KEY])
for k in range(num_scalar_targets):
this_target_name_orig = (
TARGET_NAME_TO_ORIG[DEFAULT_SCALAR_TARGET_NAMES[k]])
scalar_target_matrix[:, k] = numpy.array(
dataset_object.variables[this_target_name_orig][:], dtype=float)
for k in range(num_vector_targets):
this_target_name_orig = (
TARGET_NAME_TO_ORIG[DEFAULT_VECTOR_TARGET_NAMES[k]])
vector_target_matrix[..., k] = numpy.array(
dataset_object.variables[this_target_name_orig][:], dtype=float)
example_dict.update({
example_utils.SCALAR_PREDICTOR_VALS_KEY:
scalar_predictor_matrix,
example_utils.VECTOR_PREDICTOR_VALS_KEY:
vector_predictor_matrix,
example_utils.SCALAR_TARGET_VALS_KEY:
scalar_target_matrix,
example_utils.VECTOR_TARGET_VALS_KEY:
vector_target_matrix
})
dataset_object.close()
example_dict[example_utils.EXAMPLE_IDS_KEY] = (
example_utils.create_example_ids(example_dict))
if allow_bad_values:
bad_predictor_flags = numpy.logical_or(
numpy.any(scalar_predictor_matrix >= MIN_BAD_VALUE, axis=1),
numpy.any(vector_predictor_matrix >= MIN_BAD_VALUE, axis=(1, 2)))
bad_target_flags = numpy.logical_or(
numpy.any(scalar_target_matrix >= MIN_BAD_VALUE, axis=1),
numpy.any(vector_target_matrix >= MIN_BAD_VALUE, axis=(1, 2)))
good_indices = numpy.where(
numpy.invert(
numpy.logical_or(bad_predictor_flags, bad_target_flags)))[0]
num_examples = scalar_predictor_matrix.shape[0]
if len(good_indices) != num_examples:
warning_string = '{0:d} of {1:d} examples have bad values.'.format(
num_examples - len(good_indices), num_examples)
warnings.warn(warning_string)
example_dict = example_utils.subset_by_index(
example_dict=example_dict, desired_indices=good_indices)
longitude_index = (
example_dict[example_utils.SCALAR_PREDICTOR_NAMES_KEY].index(
example_utils.LONGITUDE_NAME))
k = example_utils.SCALAR_PREDICTOR_VALS_KEY
example_dict[k][:, longitude_index] = (
longitude_conv.convert_lng_positive_in_west(
longitudes_deg=example_dict[k][:, longitude_index],
allow_nan=False))
example_dict = _get_water_path_profiles(example_dict=example_dict,
get_lwp=True,
get_iwp=True,
get_wvp=True,
integrate_upward=False)
example_dict = _get_water_path_profiles(example_dict=example_dict,
get_lwp=True,
get_iwp=True,
get_wvp=True,
integrate_upward=True)
example_dict = _specific_to_relative_humidity(example_dict)
example_dict = example_utils.fluxes_actual_to_increments(example_dict)
return example_utils.fluxes_increments_to_actual(example_dict)
def _fluxes_increments_to_actual(vector_target_matrix,
vector_prediction_matrix,
model_metadata_dict):
"""If necessary, converts flux increments to actual fluxes.
This method is a wrapper for `example_utils.fluxes_increments_to_actual`.
:param vector_target_matrix: numpy array with actual target values.
:param vector_prediction_matrix: numpy array with predicted values.
:param model_metadata_dict: Dictionary read by `neural_net.read_metafile`.
:return: vector_target_matrix: Same as input but with different
shape/values.
:return: vector_prediction_matrix: Same as input but with different
shape/values.
:return: model_metadata_dict: Same as input but with different list of
vector target variables.
"""
generator_option_dict = model_metadata_dict[
neural_net.TRAINING_OPTIONS_KEY]
vector_target_names = (
generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY])
need_fluxes = not all([t in vector_target_names for t in FLUX_NAMES])
have_flux_increments = all(
[t in vector_target_names for t in FLUX_INCREMENT_NAMES])
if not (need_fluxes and have_flux_increments):
return (vector_target_matrix, vector_prediction_matrix,
model_metadata_dict)
num_examples = vector_target_matrix.shape[0]
num_heights = vector_target_matrix.shape[1]
base_example_dict = {
example_utils.SCALAR_PREDICTOR_NAMES_KEY: [],
example_utils.SCALAR_PREDICTOR_VALS_KEY:
numpy.full((num_examples, 0), 0.),
example_utils.VECTOR_PREDICTOR_NAMES_KEY: [],
example_utils.VECTOR_PREDICTOR_VALS_KEY:
numpy.full((num_examples, num_heights, 0), 0.),
example_utils.SCALAR_TARGET_NAMES_KEY: [],
example_utils.SCALAR_TARGET_VALS_KEY:
numpy.full((num_examples, 0), 0.),
example_utils.VECTOR_TARGET_NAMES_KEY:
generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY],
example_utils.HEIGHTS_KEY:
generator_option_dict[neural_net.HEIGHTS_KEY],
example_utils.VALID_TIMES_KEY:
numpy.full(num_examples, 0, dtype=int)
}
target_example_dict = copy.deepcopy(base_example_dict)
target_example_dict[example_utils.VECTOR_TARGET_VALS_KEY] = (
vector_target_matrix)
target_example_dict = example_utils.fluxes_increments_to_actual(
target_example_dict)
vector_target_matrix = (
target_example_dict[example_utils.VECTOR_TARGET_VALS_KEY])
prediction_example_dict = copy.deepcopy(base_example_dict)
prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY] = (
vector_prediction_matrix)
prediction_example_dict = example_utils.fluxes_increments_to_actual(
prediction_example_dict)
vector_prediction_matrix = (
prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY])
generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY] = (
target_example_dict[example_utils.VECTOR_TARGET_NAMES_KEY])
model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY] = (
generator_option_dict)
return vector_target_matrix, vector_prediction_matrix, model_metadata_dict