def test_normalize_data_minmax_targets_with_height(self): """Ensures correct output from normalize_data. In this case, using min-max normalization for targets only, with separation by height. """ this_example_dict = normalization.normalize_data( new_example_dict=copy.deepcopy(EXAMPLE_DICT_ORIG), training_example_dict=TRAINING_EXAMPLE_DICT, normalization_type_string=normalization.MINMAX_NORM_STRING, min_normalized_value=MIN_NORMALIZED_VALUE, max_normalized_value=MAX_NORMALIZED_VALUE, separate_heights=True, apply_to_predictors=False, apply_to_vector_targets=True, apply_to_scalar_targets=True) self.assertTrue( _compare_example_dicts(this_example_dict, EXAMPLE_DICT_MINMAX_TARGETS_WITH_HEIGHT)) this_example_dict = normalization.denormalize_data( new_example_dict=copy.deepcopy( EXAMPLE_DICT_MINMAX_TARGETS_WITH_HEIGHT), training_example_dict=TRAINING_EXAMPLE_DICT, normalization_type_string=normalization.MINMAX_NORM_STRING, min_normalized_value=MIN_NORMALIZED_VALUE, max_normalized_value=MAX_NORMALIZED_VALUE, separate_heights=True, apply_to_predictors=False, apply_to_vector_targets=True, apply_to_scalar_targets=True) self.assertTrue( _compare_example_dicts(this_example_dict, EXAMPLE_DICT_DENORM_TARGETS_WITH_HEIGHT))
def _run(model_file_name, example_dir_name, first_time_string, last_time_string, exclude_summit_greenland, output_file_name): """Applies trained neural net in inference mode. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_dir_name: Same. :param first_time_string: Same. :param last_time_string: Same. :param exclude_summit_greenland: Same. :param output_file_name: Same. """ first_time_unix_sec = time_conversion.string_to_unix_sec( first_time_string, TIME_FORMAT) last_time_unix_sec = time_conversion.string_to_unix_sec( last_time_string, TIME_FORMAT) print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) generator_option_dict = copy.deepcopy( metadata_dict[neural_net.TRAINING_OPTIONS_KEY]) generator_option_dict[neural_net.EXAMPLE_DIRECTORY_KEY] = example_dir_name generator_option_dict[neural_net.FIRST_TIME_KEY] = first_time_unix_sec generator_option_dict[neural_net.LAST_TIME_KEY] = last_time_unix_sec vector_target_norm_type_string = copy.deepcopy( generator_option_dict[neural_net.VECTOR_TARGET_NORM_TYPE_KEY]) scalar_target_norm_type_string = copy.deepcopy( generator_option_dict[neural_net.SCALAR_TARGET_NORM_TYPE_KEY]) generator_option_dict[neural_net.VECTOR_TARGET_NORM_TYPE_KEY] = None generator_option_dict[neural_net.SCALAR_TARGET_NORM_TYPE_KEY] = None net_type_string = metadata_dict[neural_net.NET_TYPE_KEY] predictor_matrix, target_array, example_id_strings = neural_net.create_data( option_dict=generator_option_dict, for_inference=True, net_type_string=net_type_string, exclude_summit_greenland=exclude_summit_greenland) print(SEPARATOR_STRING) exec_start_time_unix_sec = time.time() prediction_array = neural_net.apply_model( model_object=model_object, predictor_matrix=predictor_matrix, num_examples_per_batch=NUM_EXAMPLES_PER_BATCH, net_type_string=net_type_string, verbose=True) print(SEPARATOR_STRING) print('Time to apply neural net = {0:.4f} seconds'.format( time.time() - exec_start_time_unix_sec)) vector_target_matrix = target_array[0] vector_prediction_matrix = prediction_array[0] if len(target_array) == 2: scalar_target_matrix = target_array[1] scalar_prediction_matrix = prediction_array[1] else: scalar_target_matrix = None scalar_prediction_matrix = None target_example_dict = _targets_numpy_to_dict( scalar_target_matrix=scalar_target_matrix, vector_target_matrix=vector_target_matrix, model_metadata_dict=metadata_dict) prediction_example_dict = _targets_numpy_to_dict( scalar_target_matrix=scalar_prediction_matrix, vector_target_matrix=vector_prediction_matrix, model_metadata_dict=metadata_dict) normalization_file_name = ( generator_option_dict[neural_net.NORMALIZATION_FILE_KEY]) print(('Reading training examples (for normalization) from: "{0:s}"...' ).format(normalization_file_name)) training_example_dict = example_io.read_file(normalization_file_name) training_example_dict = example_utils.subset_by_height( example_dict=training_example_dict, heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY]) num_examples = len(example_id_strings) num_heights = len(prediction_example_dict[example_utils.HEIGHTS_KEY]) this_dict = { example_utils.VECTOR_PREDICTOR_NAMES_KEY: [], example_utils.VECTOR_PREDICTOR_VALS_KEY: numpy.full((num_examples, num_heights, 0), 0.), example_utils.SCALAR_PREDICTOR_NAMES_KEY: [], example_utils.SCALAR_PREDICTOR_VALS_KEY: numpy.full((num_examples, 0), 0.) } target_example_dict.update(this_dict) prediction_example_dict.update(this_dict) if vector_target_norm_type_string is not None: print('Denormalizing predicted vectors...') # down_flux_inc_matrix_w_m03 = example_utils.get_field_from_dict( # example_dict=prediction_example_dict, # field_name=example_utils.SHORTWAVE_DOWN_FLUX_INC_NAME # ) # print(down_flux_inc_matrix_w_m03[0, ...]) # print('\n') prediction_example_dict = normalization.denormalize_data( new_example_dict=prediction_example_dict, training_example_dict=training_example_dict, normalization_type_string=vector_target_norm_type_string, min_normalized_value=generator_option_dict[ neural_net.VECTOR_TARGET_MIN_VALUE_KEY], max_normalized_value=generator_option_dict[ neural_net.VECTOR_TARGET_MAX_VALUE_KEY], separate_heights=True, apply_to_predictors=False, apply_to_vector_targets=True, apply_to_scalar_targets=False) # down_flux_inc_matrix_w_m03 = example_utils.get_field_from_dict( # example_dict=prediction_example_dict, # field_name=example_utils.SHORTWAVE_DOWN_FLUX_INC_NAME # ) # print(down_flux_inc_matrix_w_m03[0, ...]) # print('\n\n\n') if scalar_target_norm_type_string is not None: print('Denormalizing predicted scalars...') prediction_example_dict = normalization.denormalize_data( new_example_dict=prediction_example_dict, training_example_dict=training_example_dict, normalization_type_string=scalar_target_norm_type_string, min_normalized_value=generator_option_dict[ neural_net.SCALAR_TARGET_MIN_VALUE_KEY], max_normalized_value=generator_option_dict[ neural_net.SCALAR_TARGET_MAX_VALUE_KEY], separate_heights=True, apply_to_predictors=False, apply_to_vector_targets=False, apply_to_scalar_targets=True) add_heating_rate = generator_option_dict[neural_net.OMIT_HEATING_RATE_KEY] if add_heating_rate: pressure_matrix_pascals = _get_unnormalized_pressure( model_metadata_dict=metadata_dict, example_id_strings=example_id_strings) prediction_example_dict = _get_predicted_heating_rates( prediction_example_dict=prediction_example_dict, pressure_matrix_pascals=pressure_matrix_pascals, model_metadata_dict=metadata_dict) vector_target_names = ( generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY]) if example_utils.SHORTWAVE_HEATING_RATE_NAME in vector_target_names: heating_rate_index = vector_target_names.index( example_utils.SHORTWAVE_HEATING_RATE_NAME) heights_m_agl = generator_option_dict[neural_net.HEIGHTS_KEY] height_indices = numpy.where( heights_m_agl >= ZERO_HEATING_HEIGHT_M_AGL)[0] vector_target_matrix = ( prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY]) vector_target_matrix[..., heating_rate_index][..., height_indices] = 0. prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY] = ( vector_target_matrix) all_heights_m_agl = generator_option_dict[neural_net.HEIGHTS_KEY] desired_heights_m_agl = ( all_heights_m_agl[all_heights_m_agl < MAX_HEIGHT_M_AGL]) target_example_dict = example_utils.subset_by_height( example_dict=target_example_dict, heights_m_agl=desired_heights_m_agl) prediction_example_dict = example_utils.subset_by_height( example_dict=prediction_example_dict, heights_m_agl=desired_heights_m_agl) print('Writing target (actual) and predicted values to: "{0:s}"...'.format( output_file_name)) prediction_io.write_file(netcdf_file_name=output_file_name, scalar_target_matrix=target_example_dict[ example_utils.SCALAR_TARGET_VALS_KEY], vector_target_matrix=target_example_dict[ example_utils.VECTOR_TARGET_VALS_KEY], scalar_prediction_matrix=prediction_example_dict[ example_utils.SCALAR_TARGET_VALS_KEY], vector_prediction_matrix=prediction_example_dict[ example_utils.VECTOR_TARGET_VALS_KEY], heights_m_agl=desired_heights_m_agl, example_id_strings=example_id_strings, model_file_name=model_file_name)
def _run(model_file_name, example_file_name, num_examples, example_dir_name, example_id_file_name, layer_name, neuron_indices, ideal_activation, num_iterations, learning_rate, l2_weight, output_file_name): """Runs backwards optimization. This is effectively the main method. :param model_file_name: See documentation at top of file. :param example_file_name: Same. :param num_examples: Same. :param example_dir_name: Same. :param example_id_file_name: Same. :param layer_name: Same. :param neuron_indices: Same. :param ideal_activation: Same. :param num_iterations: Same. :param learning_rate: Same. :param l2_weight: Same. :param output_file_name: Same. """ print('Reading model from: "{0:s}"...'.format(model_file_name)) model_object = neural_net.read_model(model_file_name) metafile_name = neural_net.find_metafile( model_dir_name=os.path.split(model_file_name)[0], raise_error_if_missing=True ) print('Reading metadata from: "{0:s}"...'.format(metafile_name)) metadata_dict = neural_net.read_metafile(metafile_name) predictor_matrix, _, example_id_strings = ( misc_utils.get_examples_for_inference( model_metadata_dict=metadata_dict, example_file_name=example_file_name, num_examples=num_examples, example_dir_name=example_dir_name, example_id_file_name=example_id_file_name ) ) print(SEPARATOR_STRING) generator_option_dict = metadata_dict[neural_net.TRAINING_OPTIONS_KEY] normalization_file_name = ( generator_option_dict[neural_net.NORMALIZATION_FILE_KEY] ) print(( 'Reading training examples (for normalization) from: "{0:s}"...' ).format( normalization_file_name )) training_example_dict = example_io.read_file(normalization_file_name) training_example_dict = example_utils.subset_by_height( example_dict=training_example_dict, heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY] ) num_examples = len(example_id_strings) bwo_dict = None for i in range(num_examples): this_bwo_dict = bwo.optimize_input_for_neuron( model_object=model_object, init_function_or_matrix=predictor_matrix[i, ...], layer_name=layer_name, neuron_indices=neuron_indices, ideal_activation=ideal_activation, num_iterations=num_iterations, learning_rate=learning_rate, l2_weight=l2_weight ) if i == num_examples - 1: print(SEPARATOR_STRING) else: print(MINOR_SEPARATOR_STRING) if bwo_dict is None: these_dim = numpy.array( (num_examples,) + this_bwo_dict[bwo.INITIAL_PREDICTORS_KEY].shape[1:], dtype=int ) bwo_dict = { bwo.INITIAL_PREDICTORS_KEY: numpy.full(these_dim, numpy.nan), bwo.FINAL_PREDICTORS_KEY: numpy.full(these_dim, numpy.nan), bwo.INITIAL_ACTIVATIONS_KEY: numpy.full(num_examples, numpy.nan), bwo.FINAL_ACTIVATIONS_KEY: numpy.full(num_examples, numpy.nan) } bwo_dict[bwo.INITIAL_PREDICTORS_KEY][i, ...] = ( this_bwo_dict[bwo.INITIAL_PREDICTORS_KEY][0, ...] ) bwo_dict[bwo.FINAL_PREDICTORS_KEY][i, ...] = ( this_bwo_dict[bwo.FINAL_PREDICTORS_KEY][0, ...] ) bwo_dict[bwo.INITIAL_ACTIVATIONS_KEY][i] = ( this_bwo_dict[bwo.INITIAL_ACTIVATION_KEY] ) bwo_dict[bwo.FINAL_ACTIVATIONS_KEY][i] = ( this_bwo_dict[bwo.FINAL_ACTIVATION_KEY] ) if example_file_name == '': example_file_name = example_io.find_many_files( directory_name=example_dir_name, first_time_unix_sec=0, last_time_unix_sec=int(1e12), raise_error_if_any_missing=False, raise_error_if_all_missing=True )[0] first_example_dict = example_io.read_file(example_file_name) first_example_dict = example_utils.subset_by_height( example_dict=first_example_dict, heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY] ) net_type_string = metadata_dict[neural_net.NET_TYPE_KEY] init_example_dict = copy.deepcopy(first_example_dict) this_example_dict = neural_net.predictors_numpy_to_dict( predictor_matrix=bwo_dict[bwo.INITIAL_PREDICTORS_KEY], example_dict=init_example_dict, net_type_string=net_type_string ) init_example_dict.update(this_example_dict) if generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY] is not None: init_example_dict = normalization.denormalize_data( new_example_dict=init_example_dict, training_example_dict=training_example_dict, normalization_type_string= generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY], min_normalized_value= generator_option_dict[neural_net.PREDICTOR_MIN_NORM_VALUE_KEY], max_normalized_value= generator_option_dict[neural_net.PREDICTOR_MAX_NORM_VALUE_KEY], separate_heights=True, apply_to_predictors=True, apply_to_vector_targets=False, apply_to_scalar_targets=False ) init_scalar_predictor_matrix = ( init_example_dict[example_utils.SCALAR_PREDICTOR_VALS_KEY] ) init_vector_predictor_matrix = ( init_example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] ) final_example_dict = copy.deepcopy(first_example_dict) this_example_dict = neural_net.predictors_numpy_to_dict( predictor_matrix=bwo_dict[bwo.FINAL_PREDICTORS_KEY], example_dict=final_example_dict, net_type_string=net_type_string ) final_example_dict.update(this_example_dict) if generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY] is not None: final_example_dict = normalization.denormalize_data( new_example_dict=final_example_dict, training_example_dict=training_example_dict, normalization_type_string= generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY], min_normalized_value= generator_option_dict[neural_net.PREDICTOR_MIN_NORM_VALUE_KEY], max_normalized_value= generator_option_dict[neural_net.PREDICTOR_MAX_NORM_VALUE_KEY], separate_heights=True, apply_to_predictors=True, apply_to_vector_targets=False, apply_to_scalar_targets=False ) final_scalar_predictor_matrix = ( final_example_dict[example_utils.SCALAR_PREDICTOR_VALS_KEY] ) final_vector_predictor_matrix = ( final_example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY] ) print('Writing results to file: "{0:s}"...'.format(output_file_name)) bwo.write_file( netcdf_file_name=output_file_name, init_scalar_predictor_matrix=init_scalar_predictor_matrix, final_scalar_predictor_matrix=final_scalar_predictor_matrix, init_vector_predictor_matrix=init_vector_predictor_matrix, final_vector_predictor_matrix=final_vector_predictor_matrix, initial_activations=bwo_dict[bwo.INITIAL_ACTIVATIONS_KEY], final_activations=bwo_dict[bwo.FINAL_ACTIVATIONS_KEY], example_id_strings=example_id_strings, model_file_name=model_file_name, layer_name=layer_name, neuron_indices=neuron_indices, ideal_activation=ideal_activation, num_iterations=num_iterations, learning_rate=learning_rate, l2_weight=l2_weight )