def _layerwise_water_path_to_content(layerwise_path_matrix_kg_m02, heights_m_agl): """Converts profile of layerwise water path to profile of water content. E = number of examples H = number of heights :param layerwise_path_matrix_kg_m02: E-by-H numpy array of layerwise water paths (kg m^-2). "Layerwise" means that the value in each grid cell is only the water path through that grid cell, not integrated over multiple grid cells. :param heights_m_agl: length-H numpy array with heights of grid-cell centers (metres above ground level). :return: water_content_matrix_kg_m03: E-by-H numpy array of water contents (kg m^-3). """ edge_heights_m_agl = example_utils.get_grid_cell_edges(heights_m_agl) grid_cell_widths_metres = example_utils.get_grid_cell_widths( edge_heights_m_agl) num_examples = layerwise_path_matrix_kg_m02.shape[0] num_heights = layerwise_path_matrix_kg_m02.shape[1] grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres, (1, num_heights)) grid_cell_width_matrix_metres = numpy.repeat(grid_cell_width_matrix_metres, repeats=num_examples, axis=0) return layerwise_path_matrix_kg_m02 / grid_cell_width_matrix_metres
def _water_content_to_layerwise_path(water_content_matrix_kg_m03, heights_m_agl): """Converts profile of water content to layerwise water path. This method is the inverse of `_layerwise_water_path_to_content`. :param water_content_matrix_kg_m03: See doc for `_layerwise_water_path_to_content`. :param heights_m_agl: Same. :return: layerwise_path_matrix_kg_m02: Same. """ edge_heights_m_agl = example_utils.get_grid_cell_edges(heights_m_agl) grid_cell_widths_metres = example_utils.get_grid_cell_widths( edge_heights_m_agl) num_examples = water_content_matrix_kg_m03.shape[0] num_heights = water_content_matrix_kg_m03.shape[1] grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres, (1, num_heights)) grid_cell_width_matrix_metres = numpy.repeat(grid_cell_width_matrix_metres, repeats=num_examples, axis=0) return water_content_matrix_kg_m03 * grid_cell_width_matrix_metres
def test_get_grid_cell_widths(self): """Ensures correct output from get_grid_cell_widths.""" these_widths_metres = ( example_utils.get_grid_cell_widths(EDGE_HEIGHTS_M_AGL)) self.assertTrue( numpy.allclose(these_widths_metres, GRID_CELL_WIDTHS_METRES, atol=TOLERANCE))
def flux_increment_loss_not_dense(up_flux_inc_channel_index, down_flux_inc_channel_index, net_flux_increment_weight, total_net_flux_weight, use_magnitude_weight, heights_m_agl): """Loss function for non-dense net that predict flux increments. "Flux increments" are DF_down / Dz and DF_up / Dz. :param up_flux_inc_channel_index: Channel index for upwelling-flux increment (DF_up). :param down_flux_inc_channel_index: Channel index for downwelling-flux increment (DF_down). :param net_flux_increment_weight: Weight for mean squared error (MSE) between predicted and actual net-flux increments (DF_net / Dz). :param total_net_flux_weight: Weight for MSE between predicted and actual net fluxes (F_net). :param use_magnitude_weight: Boolean flag. If True, the loss for each element (each example at each height) will be weighted by the magnitude of DF_net / Dz (max between predicted and actual). :param heights_m_agl: 1-D numpy array of heights in profile (metres above ground level). :return: loss: Loss function (defined below). """ # TODO(thunderhoser): This loss function should be used only when there are # two target variables, unnormalized downwelling- and upwelling-flux # increments. # TODO(thunderhoser): In the future, I may want to use fictitious Dp # (pressure increment) to convert fluxes to heating rates, then directly # penalize heating rates. error_checking.assert_is_integer(up_flux_inc_channel_index) error_checking.assert_is_geq(up_flux_inc_channel_index, 0) error_checking.assert_is_integer(down_flux_inc_channel_index) error_checking.assert_is_geq(down_flux_inc_channel_index, 0) assert up_flux_inc_channel_index != down_flux_inc_channel_index error_checking.assert_is_geq(net_flux_increment_weight, 0.) error_checking.assert_is_geq(total_net_flux_weight, 0.) error_checking.assert_is_greater( net_flux_increment_weight + total_net_flux_weight, 0.) error_checking.assert_is_boolean(use_magnitude_weight) edge_heights_m_agl = example_utils.get_grid_cell_edges(heights_m_agl) grid_cell_widths_metres = ( example_utils.get_grid_cell_widths(edge_heights_m_agl)) num_heights = len(heights_m_agl) grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres, (1, num_heights)) print(grid_cell_width_matrix_metres) def loss(target_tensor, prediction_tensor): """Computes loss. :param target_tensor: Tensor of target (actual) values. :param prediction_tensor: Tensor of predicted values. :return: loss: Scalar. """ target_net_flux_inc_tensor_w_m03 = ( target_tensor[..., down_flux_inc_channel_index] - target_tensor[..., up_flux_inc_channel_index]) predicted_net_flux_inc_tensor_w_m03 = ( prediction_tensor[..., down_flux_inc_channel_index] - prediction_tensor[..., up_flux_inc_channel_index]) loss = net_flux_increment_weight * ( predicted_net_flux_inc_tensor_w_m03 - target_net_flux_inc_tensor_w_m03)**2 target_net_flux_tensor_w_m02 = K.cumsum( target_net_flux_inc_tensor_w_m03 * grid_cell_width_matrix_metres, axis=1) predicted_net_flux_tensor_w_m02 = K.cumsum( predicted_net_flux_inc_tensor_w_m03 * grid_cell_width_matrix_metres, axis=1) loss += total_net_flux_weight * (predicted_net_flux_tensor_w_m02 - target_net_flux_tensor_w_m02)**2 if use_magnitude_weight: loss = loss * K.maximum(target_net_flux_inc_tensor_w_m03, predicted_net_flux_inc_tensor_w_m03) return K.mean(loss) return loss
def flux_increment_loss_dense(first_up_flux_inc_index, first_down_flux_inc_index, net_flux_increment_weight, total_net_flux_weight, use_magnitude_weight, heights_m_agl): """Loss function for dense net that predict flux increments. :param first_up_flux_inc_index: Array index for upwelling-flux increment at lowest height. :param first_down_flux_inc_index: Array index for downwelling-flux increment at lowest height. :param net_flux_increment_weight: See doc for `flux_increment_loss_not_dense`. :param total_net_flux_weight: Same. :param use_magnitude_weight: Same. :param heights_m_agl: Same. :return: loss: Loss function (defined below). """ error_checking.assert_is_integer(first_up_flux_inc_index) error_checking.assert_is_geq(first_up_flux_inc_index, 0) error_checking.assert_is_integer(first_down_flux_inc_index) error_checking.assert_is_geq(first_down_flux_inc_index, 0) error_checking.assert_is_geq(net_flux_increment_weight, 0.) error_checking.assert_is_geq(total_net_flux_weight, 0.) error_checking.assert_is_greater( net_flux_increment_weight + total_net_flux_weight, 0.) error_checking.assert_is_boolean(use_magnitude_weight) edge_heights_m_agl = example_utils.get_grid_cell_edges(heights_m_agl) grid_cell_widths_metres = ( example_utils.get_grid_cell_widths(edge_heights_m_agl)) num_heights = len(heights_m_agl) grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres, (1, num_heights)) def loss(target_tensor, prediction_tensor): """Computes loss. :param target_tensor: Tensor of target (actual) values. :param prediction_tensor: Tensor of predicted values. :return: loss: Scalar. """ j = first_down_flux_inc_index k = first_up_flux_inc_index target_net_flux_inc_tensor_w_m03 = ( target_tensor[..., j:(j + num_heights)] - target_tensor[..., k:(k + num_heights)]) predicted_net_flux_inc_tensor_w_m03 = ( prediction_tensor[..., j:(j + num_heights)] - prediction_tensor[..., k:(k + num_heights)]) loss = net_flux_increment_weight * ( predicted_net_flux_inc_tensor_w_m03 - target_net_flux_inc_tensor_w_m03)**2 target_net_flux_tensor_w_m02 = K.cumsum( target_net_flux_inc_tensor_w_m03 * grid_cell_width_matrix_metres, axis=1) predicted_net_flux_tensor_w_m02 = K.cumsum( predicted_net_flux_inc_tensor_w_m03 * grid_cell_width_matrix_metres, axis=1) loss += total_net_flux_weight * (predicted_net_flux_tensor_w_m02 - target_net_flux_tensor_w_m02)**2 if use_magnitude_weight: loss = loss * K.maximum(target_net_flux_inc_tensor_w_m03, predicted_net_flux_inc_tensor_w_m03) return K.mean(loss) return loss
def _get_water_path_profiles(example_dict, get_lwp=True, get_iwp=True, get_wvp=True, integrate_upward=False): """Computes profiles of LWP, IWP, and/or WVP. LWP = liquid-water path IWP = ice-water path WVP = water-vapour path If `integrate_upward == False`, then at height z, the LWP/IWP/WVP is the integral of LWC/IWC/WVC (respectively) from the top of atmosphere to z. If `integrate_upward == True`, then at height z, the LWP/IWP/WVP is the integral of LWC/IWC/WVC (respectively) from the surface to z. :param example_dict: Dictionary of examples (in the format returned by `read_file`). :param get_lwp: Boolean flag. If True, will compute LWP profile for each example. :param get_iwp: Boolean flag. If True, will compute IWP profile for each example. :param get_wvp: Boolean flag. If True, will compute WVP profile for each example. :param integrate_upward: Boolean flag. If True, will integrate from the surface up. If False, will integrate from the top of atmosphere down. :return: example_dict: Same as input but with extra predictor variables. """ vector_predictor_names = ( example_dict[example_utils.VECTOR_PREDICTOR_NAMES_KEY]) if integrate_upward: this_liquid_path_name = example_utils.UPWARD_LIQUID_WATER_PATH_NAME this_ice_path_name = example_utils.UPWARD_ICE_WATER_PATH_NAME this_vapour_path_name = example_utils.UPWARD_WATER_VAPOUR_PATH_NAME else: this_liquid_path_name = example_utils.LIQUID_WATER_PATH_NAME this_ice_path_name = example_utils.ICE_WATER_PATH_NAME this_vapour_path_name = example_utils.WATER_VAPOUR_PATH_NAME get_lwp = get_lwp and this_liquid_path_name not in vector_predictor_names get_iwp = get_iwp and this_ice_path_name not in vector_predictor_names get_wvp = get_wvp and this_vapour_path_name not in vector_predictor_names if not (get_lwp or get_iwp or get_wvp): return example_dict edge_heights_m_agl = example_utils.get_grid_cell_edges( example_dict[example_utils.HEIGHTS_KEY]) grid_cell_widths_metres = example_utils.get_grid_cell_widths( edge_heights_m_agl) num_examples = len(example_dict[example_utils.VALID_TIMES_KEY]) num_heights = len(example_dict[example_utils.HEIGHTS_KEY]) grid_cell_width_matrix_metres = numpy.reshape(grid_cell_widths_metres, (1, num_heights)) grid_cell_width_matrix_metres = numpy.repeat(grid_cell_width_matrix_metres, repeats=num_examples, axis=0) if get_lwp: lwc_matrix_kg_m03 = example_utils.get_field_from_dict( example_dict=example_dict, field_name=example_utils.LIQUID_WATER_CONTENT_NAME) if integrate_upward: lwp_matrix_kg_m02 = numpy.cumsum(lwc_matrix_kg_m03 * grid_cell_width_matrix_metres, axis=1) else: lwp_matrix_kg_m02 = numpy.fliplr( numpy.cumsum(numpy.fliplr(lwc_matrix_kg_m03 * grid_cell_width_matrix_metres), axis=1)) example_dict[example_utils.VECTOR_PREDICTOR_NAMES_KEY].append( this_liquid_path_name) example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] = ( numpy.concatenate( (example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY], numpy.expand_dims(lwp_matrix_kg_m02, axis=-1)), axis=-1)) if get_iwp: iwc_matrix_kg_m03 = example_utils.get_field_from_dict( example_dict=example_dict, field_name=example_utils.ICE_WATER_CONTENT_NAME) if integrate_upward: iwp_matrix_kg_m02 = numpy.cumsum(iwc_matrix_kg_m03 * grid_cell_width_matrix_metres, axis=1) else: iwp_matrix_kg_m02 = numpy.fliplr( numpy.cumsum(numpy.fliplr(iwc_matrix_kg_m03 * grid_cell_width_matrix_metres), axis=1)) example_dict[example_utils.VECTOR_PREDICTOR_NAMES_KEY].append( this_ice_path_name) example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] = ( numpy.concatenate( (example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY], numpy.expand_dims(iwp_matrix_kg_m02, axis=-1)), axis=-1)) if get_wvp: air_density_matrix_kg_m03 = _get_air_density(example_dict) specific_humidity_matrix_kg_kg01 = example_utils.get_field_from_dict( example_dict=example_dict, field_name=example_utils.SPECIFIC_HUMIDITY_NAME) vapour_content_matrix_kg_m03 = (specific_humidity_matrix_kg_kg01 * air_density_matrix_kg_m03) if integrate_upward: vapour_path_matrix_kg_m02 = numpy.cumsum( vapour_content_matrix_kg_m03 * grid_cell_width_matrix_metres, axis=1) else: vapour_path_matrix_kg_m02 = numpy.fliplr( numpy.cumsum(numpy.fliplr(vapour_content_matrix_kg_m03 * grid_cell_width_matrix_metres), axis=1)) example_dict[example_utils.VECTOR_PREDICTOR_NAMES_KEY].append( this_vapour_path_name) example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] = ( numpy.concatenate( (example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY], numpy.expand_dims(vapour_path_matrix_kg_m02, axis=-1)), axis=-1)) return example_dict