def test_fluxes_increments_to_actual(self): """Ensures correct output from fluxes_increments_to_actual.""" this_example_dict = example_utils.fluxes_increments_to_actual( copy.deepcopy(EXAMPLE_DICT_WITH_INCREMENTS)) self.assertTrue( _compare_example_dicts(this_example_dict, EXAMPLE_DICT_WITH_INCREMENTS))
def _get_predicted_heating_rates(prediction_example_dict, pressure_matrix_pascals, model_metadata_dict): """Computes predicted heating rates from predicted flux-increment profiles. :param prediction_example_dict: Dictionary with predictions. For a list of keys, see doc for `example_io.read_file`. :param pressure_matrix_pascals: See doc for `_get_unnormalized_pressure`. :param model_metadata_dict: Same. :return: prediction_example_dict: Same but with heating rates. """ num_examples = pressure_matrix_pascals.shape[0] generator_option_dict = copy.deepcopy( model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY]) this_dict = { example_utils.VECTOR_PREDICTOR_NAMES_KEY: [example_utils.PRESSURE_NAME], example_utils.VECTOR_PREDICTOR_VALS_KEY: numpy.expand_dims(pressure_matrix_pascals, axis=-1), example_utils.SCALAR_PREDICTOR_NAMES_KEY: [], example_utils.SCALAR_PREDICTOR_VALS_KEY: numpy.full((num_examples, 0), 0.), example_utils.VALID_TIMES_KEY: numpy.full(num_examples, 0, dtype=int) } prediction_example_dict.update(this_dict) prediction_example_dict = ( example_utils.fluxes_increments_to_actual(prediction_example_dict)) prediction_example_dict = example_utils.fluxes_to_heating_rate( prediction_example_dict) target_names = (generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY] + generator_option_dict[neural_net.SCALAR_TARGET_NAMES_KEY]) return example_utils.subset_by_field(example_dict=prediction_example_dict, field_names=target_names)
def read_file(netcdf_file_name, allow_bad_values=False): """Reads RRTM data from NetCDF file. E = number of examples H = number of heights P_s = number of scalar predictors P_v = number of vector predictors T_s = number of scalar targets T_v = number of vector targets :param netcdf_file_name: Path to NetCDF file with learning examples. :param allow_bad_values: Boolean flag. If True, will allow bad values and remove examples that have bad values. :return: example_dict: Dictionary with the following keys. example_dict['scalar_predictor_matrix']: numpy array (E x P_s) with values of scalar predictors. example_dict['scalar_predictor_names']: list (length P_s) with names of scalar predictors. example_dict['vector_predictor_matrix']: numpy array (E x H x P_v) with values of vector predictors. example_dict['vector_predictor_names']: list (length P_v) with names of vector predictors. example_dict['scalar_target_matrix']: numpy array (E x T_s) with values of scalar targets. example_dict['scalar_target_names']: list (length T_s) with names of scalar targets. example_dict['vector_target_matrix']: numpy array (E x H x T_v) with values of vector targets. example_dict['vector_target_names']: list (length T_v) with names of vector targets. example_dict['valid_times_unix_sec']: length-E numpy array of valid times (Unix seconds). example_dict['heights_m_agl']: length-H numpy array of heights (metres above ground level). example_dict['standard_atmo_flags']: length-E numpy array of flags (each in the list `STANDARD_ATMO_ENUMS`). example_dict['example_id_strings']: length-E list of example IDs. """ error_checking.assert_is_boolean(allow_bad_values) # TODO(thunderhoser): This is a HACK. if not os.path.isfile(netcdf_file_name): netcdf_file_name = netcdf_file_name.replace('/home/ryan.lagerquist', '/home/ralager') dataset_object = netCDF4.Dataset(netcdf_file_name) example_dict = { example_utils.SCALAR_PREDICTOR_NAMES_KEY: copy.deepcopy(DEFAULT_SCALAR_PREDICTOR_NAMES), example_utils.VECTOR_PREDICTOR_NAMES_KEY: copy.deepcopy(DEFAULT_VECTOR_PREDICTOR_NAMES), example_utils.SCALAR_TARGET_NAMES_KEY: copy.deepcopy(DEFAULT_SCALAR_TARGET_NAMES), example_utils.VECTOR_TARGET_NAMES_KEY: copy.deepcopy(DEFAULT_VECTOR_TARGET_NAMES), example_utils.VALID_TIMES_KEY: numpy.array(dataset_object.variables[VALID_TIMES_KEY][:], dtype=int), example_utils.HEIGHTS_KEY: KM_TO_METRES * numpy.array(dataset_object.variables[HEIGHTS_KEY][:], dtype=float), example_utils.STANDARD_ATMO_FLAGS_KEY: numpy.array(numpy.round( dataset_object.variables[STANDARD_ATMO_FLAGS_KEY][:]), dtype=int) } num_examples = len(example_dict[example_utils.VALID_TIMES_KEY]) num_heights = len(example_dict[example_utils.HEIGHTS_KEY]) num_scalar_predictors = len(DEFAULT_SCALAR_PREDICTOR_NAMES) num_vector_predictors = len(DEFAULT_VECTOR_PREDICTOR_NAMES) num_scalar_targets = len(DEFAULT_SCALAR_TARGET_NAMES) num_vector_targets = len(DEFAULT_VECTOR_TARGET_NAMES) scalar_predictor_matrix = numpy.full((num_examples, num_scalar_predictors), numpy.nan) vector_predictor_matrix = numpy.full( (num_examples, num_heights, num_vector_predictors), numpy.nan) scalar_target_matrix = numpy.full((num_examples, num_scalar_targets), numpy.nan) vector_target_matrix = numpy.full( (num_examples, num_heights, num_vector_targets), numpy.nan) for k in range(num_scalar_predictors): this_predictor_name_orig = ( PREDICTOR_NAME_TO_ORIG[DEFAULT_SCALAR_PREDICTOR_NAMES[k]]) this_conversion_factor = ( PREDICTOR_NAME_TO_CONV_FACTOR[DEFAULT_SCALAR_PREDICTOR_NAMES[k]]) scalar_predictor_matrix[:, k] = this_conversion_factor * numpy.array( dataset_object.variables[this_predictor_name_orig][:], dtype=float) for k in range(num_vector_predictors): this_predictor_name_orig = ( PREDICTOR_NAME_TO_ORIG[DEFAULT_VECTOR_PREDICTOR_NAMES[k]]) this_conversion_factor = ( PREDICTOR_NAME_TO_CONV_FACTOR[DEFAULT_VECTOR_PREDICTOR_NAMES[k]]) vector_predictor_matrix[..., k] = this_conversion_factor * numpy.array( dataset_object.variables[this_predictor_name_orig][:], dtype=float) if DEFAULT_VECTOR_PREDICTOR_NAMES[k] in [ example_utils.LIQUID_WATER_CONTENT_NAME, example_utils.ICE_WATER_CONTENT_NAME ]: vector_predictor_matrix[..., k] = _layerwise_water_path_to_content( layerwise_path_matrix_kg_m02=vector_predictor_matrix[..., k], heights_m_agl=example_dict[example_utils.HEIGHTS_KEY]) for k in range(num_scalar_targets): this_target_name_orig = ( TARGET_NAME_TO_ORIG[DEFAULT_SCALAR_TARGET_NAMES[k]]) scalar_target_matrix[:, k] = numpy.array( dataset_object.variables[this_target_name_orig][:], dtype=float) for k in range(num_vector_targets): this_target_name_orig = ( TARGET_NAME_TO_ORIG[DEFAULT_VECTOR_TARGET_NAMES[k]]) vector_target_matrix[..., k] = numpy.array( dataset_object.variables[this_target_name_orig][:], dtype=float) example_dict.update({ example_utils.SCALAR_PREDICTOR_VALS_KEY: scalar_predictor_matrix, example_utils.VECTOR_PREDICTOR_VALS_KEY: vector_predictor_matrix, example_utils.SCALAR_TARGET_VALS_KEY: scalar_target_matrix, example_utils.VECTOR_TARGET_VALS_KEY: vector_target_matrix }) dataset_object.close() example_dict[example_utils.EXAMPLE_IDS_KEY] = ( example_utils.create_example_ids(example_dict)) if allow_bad_values: bad_predictor_flags = numpy.logical_or( numpy.any(scalar_predictor_matrix >= MIN_BAD_VALUE, axis=1), numpy.any(vector_predictor_matrix >= MIN_BAD_VALUE, axis=(1, 2))) bad_target_flags = numpy.logical_or( numpy.any(scalar_target_matrix >= MIN_BAD_VALUE, axis=1), numpy.any(vector_target_matrix >= MIN_BAD_VALUE, axis=(1, 2))) good_indices = numpy.where( numpy.invert( numpy.logical_or(bad_predictor_flags, bad_target_flags)))[0] num_examples = scalar_predictor_matrix.shape[0] if len(good_indices) != num_examples: warning_string = '{0:d} of {1:d} examples have bad values.'.format( num_examples - len(good_indices), num_examples) warnings.warn(warning_string) example_dict = example_utils.subset_by_index( example_dict=example_dict, desired_indices=good_indices) longitude_index = ( example_dict[example_utils.SCALAR_PREDICTOR_NAMES_KEY].index( example_utils.LONGITUDE_NAME)) k = example_utils.SCALAR_PREDICTOR_VALS_KEY example_dict[k][:, longitude_index] = ( longitude_conv.convert_lng_positive_in_west( longitudes_deg=example_dict[k][:, longitude_index], allow_nan=False)) example_dict = _get_water_path_profiles(example_dict=example_dict, get_lwp=True, get_iwp=True, get_wvp=True, integrate_upward=False) example_dict = _get_water_path_profiles(example_dict=example_dict, get_lwp=True, get_iwp=True, get_wvp=True, integrate_upward=True) example_dict = _specific_to_relative_humidity(example_dict) example_dict = example_utils.fluxes_actual_to_increments(example_dict) return example_utils.fluxes_increments_to_actual(example_dict)
def _fluxes_increments_to_actual(vector_target_matrix, vector_prediction_matrix, model_metadata_dict): """If necessary, converts flux increments to actual fluxes. This method is a wrapper for `example_utils.fluxes_increments_to_actual`. :param vector_target_matrix: numpy array with actual target values. :param vector_prediction_matrix: numpy array with predicted values. :param model_metadata_dict: Dictionary read by `neural_net.read_metafile`. :return: vector_target_matrix: Same as input but with different shape/values. :return: vector_prediction_matrix: Same as input but with different shape/values. :return: model_metadata_dict: Same as input but with different list of vector target variables. """ generator_option_dict = model_metadata_dict[ neural_net.TRAINING_OPTIONS_KEY] vector_target_names = ( generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY]) need_fluxes = not all([t in vector_target_names for t in FLUX_NAMES]) have_flux_increments = all( [t in vector_target_names for t in FLUX_INCREMENT_NAMES]) if not (need_fluxes and have_flux_increments): return (vector_target_matrix, vector_prediction_matrix, model_metadata_dict) num_examples = vector_target_matrix.shape[0] num_heights = vector_target_matrix.shape[1] base_example_dict = { example_utils.SCALAR_PREDICTOR_NAMES_KEY: [], example_utils.SCALAR_PREDICTOR_VALS_KEY: numpy.full((num_examples, 0), 0.), example_utils.VECTOR_PREDICTOR_NAMES_KEY: [], example_utils.VECTOR_PREDICTOR_VALS_KEY: numpy.full((num_examples, num_heights, 0), 0.), example_utils.SCALAR_TARGET_NAMES_KEY: [], example_utils.SCALAR_TARGET_VALS_KEY: numpy.full((num_examples, 0), 0.), example_utils.VECTOR_TARGET_NAMES_KEY: generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY], example_utils.HEIGHTS_KEY: generator_option_dict[neural_net.HEIGHTS_KEY], example_utils.VALID_TIMES_KEY: numpy.full(num_examples, 0, dtype=int) } target_example_dict = copy.deepcopy(base_example_dict) target_example_dict[example_utils.VECTOR_TARGET_VALS_KEY] = ( vector_target_matrix) target_example_dict = example_utils.fluxes_increments_to_actual( target_example_dict) vector_target_matrix = ( target_example_dict[example_utils.VECTOR_TARGET_VALS_KEY]) prediction_example_dict = copy.deepcopy(base_example_dict) prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY] = ( vector_prediction_matrix) prediction_example_dict = example_utils.fluxes_increments_to_actual( prediction_example_dict) vector_prediction_matrix = ( prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY]) generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY] = ( target_example_dict[example_utils.VECTOR_TARGET_NAMES_KEY]) model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY] = ( generator_option_dict) return vector_target_matrix, vector_prediction_matrix, model_metadata_dict