def _layerwise_water_path_to_content(layerwise_path_matrix_kg_m02,
heights_m_agl):
"""Converts profile of layerwise water path to profile of water content.
E = number of examples
H = number of heights
:param layerwise_path_matrix_kg_m02: E-by-H numpy array of layerwise water
paths (kg m^-2). "Layerwise" means that the value in each grid cell is
only the water path through that grid cell, not integrated over multiple
grid cells.
:param heights_m_agl: length-H numpy array with heights of grid-cell centers
(metres above ground level).
:return: water_content_matrix_kg_m03: E-by-H numpy array of water contents
(kg m^-3).
"""
edge_heights_m_agl = example_utils.get_grid_cell_edges(heights_m_agl)
grid_cell_widths_metres = example_utils.get_grid_cell_widths(
edge_heights_m_agl)
num_examples = layerwise_path_matrix_kg_m02.shape[0]
num_heights = layerwise_path_matrix_kg_m02.shape[1]
grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres,
(1, num_heights))
grid_cell_width_matrix_metres = numpy.repeat(grid_cell_width_matrix_metres,
repeats=num_examples,
axis=0)
return layerwise_path_matrix_kg_m02 / grid_cell_width_matrix_metres
def _water_content_to_layerwise_path(water_content_matrix_kg_m03,
heights_m_agl):
"""Converts profile of water content to layerwise water path.
This method is the inverse of `_layerwise_water_path_to_content`.
:param water_content_matrix_kg_m03: See doc for
`_layerwise_water_path_to_content`.
:param heights_m_agl: Same.
:return: layerwise_path_matrix_kg_m02: Same.
"""
edge_heights_m_agl = example_utils.get_grid_cell_edges(heights_m_agl)
grid_cell_widths_metres = example_utils.get_grid_cell_widths(
edge_heights_m_agl)
num_examples = water_content_matrix_kg_m03.shape[0]
num_heights = water_content_matrix_kg_m03.shape[1]
grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres,
(1, num_heights))
grid_cell_width_matrix_metres = numpy.repeat(grid_cell_width_matrix_metres,
repeats=num_examples,
axis=0)
return water_content_matrix_kg_m03 * grid_cell_width_matrix_metres
def test_get_grid_cell_widths(self):
"""Ensures correct output from get_grid_cell_widths."""
these_widths_metres = (
example_utils.get_grid_cell_widths(EDGE_HEIGHTS_M_AGL))
self.assertTrue(
numpy.allclose(these_widths_metres,
GRID_CELL_WIDTHS_METRES,
atol=TOLERANCE))
def flux_increment_loss_not_dense(up_flux_inc_channel_index,
down_flux_inc_channel_index,
net_flux_increment_weight,
total_net_flux_weight, use_magnitude_weight,
heights_m_agl):
"""Loss function for non-dense net that predict flux increments.
"Flux increments" are DF_down / Dz and DF_up / Dz.
:param up_flux_inc_channel_index: Channel index for upwelling-flux increment
(DF_up).
:param down_flux_inc_channel_index: Channel index for downwelling-flux
increment (DF_down).
:param net_flux_increment_weight: Weight for mean squared error (MSE)
between predicted and actual net-flux increments (DF_net / Dz).
:param total_net_flux_weight: Weight for MSE between predicted and actual
net fluxes (F_net).
:param use_magnitude_weight: Boolean flag. If True, the loss for each
element (each example at each height) will be weighted by the magnitude
of DF_net / Dz (max between predicted and actual).
:param heights_m_agl: 1-D numpy array of heights in profile (metres above
ground level).
:return: loss: Loss function (defined below).
"""
# TODO(thunderhoser): This loss function should be used only when there are
# two target variables, unnormalized downwelling- and upwelling-flux
# increments.
# TODO(thunderhoser): In the future, I may want to use fictitious Dp
# (pressure increment) to convert fluxes to heating rates, then directly
# penalize heating rates.
error_checking.assert_is_integer(up_flux_inc_channel_index)
error_checking.assert_is_geq(up_flux_inc_channel_index, 0)
error_checking.assert_is_integer(down_flux_inc_channel_index)
error_checking.assert_is_geq(down_flux_inc_channel_index, 0)
assert up_flux_inc_channel_index != down_flux_inc_channel_index
error_checking.assert_is_geq(net_flux_increment_weight, 0.)
error_checking.assert_is_geq(total_net_flux_weight, 0.)
error_checking.assert_is_greater(
net_flux_increment_weight + total_net_flux_weight, 0.)
error_checking.assert_is_boolean(use_magnitude_weight)
edge_heights_m_agl = example_utils.get_grid_cell_edges(heights_m_agl)
grid_cell_widths_metres = (
example_utils.get_grid_cell_widths(edge_heights_m_agl))
num_heights = len(heights_m_agl)
grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres,
(1, num_heights))
print(grid_cell_width_matrix_metres)
def loss(target_tensor, prediction_tensor):
"""Computes loss.
:param target_tensor: Tensor of target (actual) values.
:param prediction_tensor: Tensor of predicted values.
:return: loss: Scalar.
"""
target_net_flux_inc_tensor_w_m03 = (
target_tensor[..., down_flux_inc_channel_index] -
target_tensor[..., up_flux_inc_channel_index])
predicted_net_flux_inc_tensor_w_m03 = (
prediction_tensor[..., down_flux_inc_channel_index] -
prediction_tensor[..., up_flux_inc_channel_index])
loss = net_flux_increment_weight * (
predicted_net_flux_inc_tensor_w_m03 -
target_net_flux_inc_tensor_w_m03)**2
target_net_flux_tensor_w_m02 = K.cumsum(
target_net_flux_inc_tensor_w_m03 * grid_cell_width_matrix_metres,
axis=1)
predicted_net_flux_tensor_w_m02 = K.cumsum(
predicted_net_flux_inc_tensor_w_m03 *
grid_cell_width_matrix_metres,
axis=1)
loss += total_net_flux_weight * (predicted_net_flux_tensor_w_m02 -
target_net_flux_tensor_w_m02)**2
if use_magnitude_weight:
loss = loss * K.maximum(target_net_flux_inc_tensor_w_m03,
predicted_net_flux_inc_tensor_w_m03)
return K.mean(loss)
return loss
def flux_increment_loss_dense(first_up_flux_inc_index,
first_down_flux_inc_index,
net_flux_increment_weight, total_net_flux_weight,
use_magnitude_weight, heights_m_agl):
"""Loss function for dense net that predict flux increments.
:param first_up_flux_inc_index: Array index for upwelling-flux increment at
lowest height.
:param first_down_flux_inc_index: Array index for downwelling-flux increment
at lowest height.
:param net_flux_increment_weight: See doc for
`flux_increment_loss_not_dense`.
:param total_net_flux_weight: Same.
:param use_magnitude_weight: Same.
:param heights_m_agl: Same.
:return: loss: Loss function (defined below).
"""
error_checking.assert_is_integer(first_up_flux_inc_index)
error_checking.assert_is_geq(first_up_flux_inc_index, 0)
error_checking.assert_is_integer(first_down_flux_inc_index)
error_checking.assert_is_geq(first_down_flux_inc_index, 0)
error_checking.assert_is_geq(net_flux_increment_weight, 0.)
error_checking.assert_is_geq(total_net_flux_weight, 0.)
error_checking.assert_is_greater(
net_flux_increment_weight + total_net_flux_weight, 0.)
error_checking.assert_is_boolean(use_magnitude_weight)
edge_heights_m_agl = example_utils.get_grid_cell_edges(heights_m_agl)
grid_cell_widths_metres = (
example_utils.get_grid_cell_widths(edge_heights_m_agl))
num_heights = len(heights_m_agl)
grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres,
(1, num_heights))
def loss(target_tensor, prediction_tensor):
"""Computes loss.
:param target_tensor: Tensor of target (actual) values.
:param prediction_tensor: Tensor of predicted values.
:return: loss: Scalar.
"""
j = first_down_flux_inc_index
k = first_up_flux_inc_index
target_net_flux_inc_tensor_w_m03 = (
target_tensor[..., j:(j + num_heights)] -
target_tensor[..., k:(k + num_heights)])
predicted_net_flux_inc_tensor_w_m03 = (
prediction_tensor[..., j:(j + num_heights)] -
prediction_tensor[..., k:(k + num_heights)])
loss = net_flux_increment_weight * (
predicted_net_flux_inc_tensor_w_m03 -
target_net_flux_inc_tensor_w_m03)**2
target_net_flux_tensor_w_m02 = K.cumsum(
target_net_flux_inc_tensor_w_m03 * grid_cell_width_matrix_metres,
axis=1)
predicted_net_flux_tensor_w_m02 = K.cumsum(
predicted_net_flux_inc_tensor_w_m03 *
grid_cell_width_matrix_metres,
axis=1)
loss += total_net_flux_weight * (predicted_net_flux_tensor_w_m02 -
target_net_flux_tensor_w_m02)**2
if use_magnitude_weight:
loss = loss * K.maximum(target_net_flux_inc_tensor_w_m03,
predicted_net_flux_inc_tensor_w_m03)
return K.mean(loss)
return loss
def _get_water_path_profiles(example_dict,
get_lwp=True,
get_iwp=True,
get_wvp=True,
integrate_upward=False):
"""Computes profiles of LWP, IWP, and/or WVP.
LWP = liquid-water path
IWP = ice-water path
WVP = water-vapour path
If `integrate_upward == False`, then at height z, the LWP/IWP/WVP is the
integral of LWC/IWC/WVC (respectively) from the top of atmosphere to z.
If `integrate_upward == True`, then at height z, the LWP/IWP/WVP is the
integral of LWC/IWC/WVC (respectively) from the surface to z.
:param example_dict: Dictionary of examples (in the format returned by
`read_file`).
:param get_lwp: Boolean flag. If True, will compute LWP profile for each
example.
:param get_iwp: Boolean flag. If True, will compute IWP profile for each
example.
:param get_wvp: Boolean flag. If True, will compute WVP profile for each
example.
:param integrate_upward: Boolean flag. If True, will integrate from the
surface up. If False, will integrate from the top of atmosphere down.
:return: example_dict: Same as input but with extra predictor variables.
"""
vector_predictor_names = (
example_dict[example_utils.VECTOR_PREDICTOR_NAMES_KEY])
if integrate_upward:
this_liquid_path_name = example_utils.UPWARD_LIQUID_WATER_PATH_NAME
this_ice_path_name = example_utils.UPWARD_ICE_WATER_PATH_NAME
this_vapour_path_name = example_utils.UPWARD_WATER_VAPOUR_PATH_NAME
else:
this_liquid_path_name = example_utils.LIQUID_WATER_PATH_NAME
this_ice_path_name = example_utils.ICE_WATER_PATH_NAME
this_vapour_path_name = example_utils.WATER_VAPOUR_PATH_NAME
get_lwp = get_lwp and this_liquid_path_name not in vector_predictor_names
get_iwp = get_iwp and this_ice_path_name not in vector_predictor_names
get_wvp = get_wvp and this_vapour_path_name not in vector_predictor_names
if not (get_lwp or get_iwp or get_wvp):
return example_dict
edge_heights_m_agl = example_utils.get_grid_cell_edges(
example_dict[example_utils.HEIGHTS_KEY])
grid_cell_widths_metres = example_utils.get_grid_cell_widths(
edge_heights_m_agl)
num_examples = len(example_dict[example_utils.VALID_TIMES_KEY])
num_heights = len(example_dict[example_utils.HEIGHTS_KEY])
grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres,
(1, num_heights))
grid_cell_width_matrix_metres = numpy.repeat(grid_cell_width_matrix_metres,
repeats=num_examples,
axis=0)
if get_lwp:
lwc_matrix_kg_m03 = example_utils.get_field_from_dict(
example_dict=example_dict,
field_name=example_utils.LIQUID_WATER_CONTENT_NAME)
if integrate_upward:
lwp_matrix_kg_m02 = numpy.cumsum(lwc_matrix_kg_m03 *
grid_cell_width_matrix_metres,
axis=1)
else:
lwp_matrix_kg_m02 = numpy.fliplr(
numpy.cumsum(numpy.fliplr(lwc_matrix_kg_m03 *
grid_cell_width_matrix_metres),
axis=1))
example_dict[example_utils.VECTOR_PREDICTOR_NAMES_KEY].append(
this_liquid_path_name)
example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] = (
numpy.concatenate(
(example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY],
numpy.expand_dims(lwp_matrix_kg_m02, axis=-1)),
axis=-1))
if get_iwp:
iwc_matrix_kg_m03 = example_utils.get_field_from_dict(
example_dict=example_dict,
field_name=example_utils.ICE_WATER_CONTENT_NAME)
if integrate_upward:
iwp_matrix_kg_m02 = numpy.cumsum(iwc_matrix_kg_m03 *
grid_cell_width_matrix_metres,
axis=1)
else:
iwp_matrix_kg_m02 = numpy.fliplr(
numpy.cumsum(numpy.fliplr(iwc_matrix_kg_m03 *
grid_cell_width_matrix_metres),
axis=1))
example_dict[example_utils.VECTOR_PREDICTOR_NAMES_KEY].append(
this_ice_path_name)
example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] = (
numpy.concatenate(
(example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY],
numpy.expand_dims(iwp_matrix_kg_m02, axis=-1)),
axis=-1))
if get_wvp:
air_density_matrix_kg_m03 = _get_air_density(example_dict)
specific_humidity_matrix_kg_kg01 = example_utils.get_field_from_dict(
example_dict=example_dict,
field_name=example_utils.SPECIFIC_HUMIDITY_NAME)
vapour_content_matrix_kg_m03 = (specific_humidity_matrix_kg_kg01 *
air_density_matrix_kg_m03)
if integrate_upward:
vapour_path_matrix_kg_m02 = numpy.cumsum(
vapour_content_matrix_kg_m03 * grid_cell_width_matrix_metres,
axis=1)
else:
vapour_path_matrix_kg_m02 = numpy.fliplr(
numpy.cumsum(numpy.fliplr(vapour_content_matrix_kg_m03 *
grid_cell_width_matrix_metres),
axis=1))
example_dict[example_utils.VECTOR_PREDICTOR_NAMES_KEY].append(
this_vapour_path_name)
example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] = (
numpy.concatenate(
(example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY],
numpy.expand_dims(vapour_path_matrix_kg_m02, axis=-1)),
axis=-1))
return example_dict