def test_normalize_data_minmax_targets_with_height(self):
"""Ensures correct output from normalize_data.
In this case, using min-max normalization for targets only, with
separation by height.
"""
this_example_dict = normalization.normalize_data(
new_example_dict=copy.deepcopy(EXAMPLE_DICT_ORIG),
training_example_dict=TRAINING_EXAMPLE_DICT,
normalization_type_string=normalization.MINMAX_NORM_STRING,
min_normalized_value=MIN_NORMALIZED_VALUE,
max_normalized_value=MAX_NORMALIZED_VALUE,
separate_heights=True,
apply_to_predictors=False,
apply_to_vector_targets=True,
apply_to_scalar_targets=True)
self.assertTrue(
_compare_example_dicts(this_example_dict,
EXAMPLE_DICT_MINMAX_TARGETS_WITH_HEIGHT))
this_example_dict = normalization.denormalize_data(
new_example_dict=copy.deepcopy(
EXAMPLE_DICT_MINMAX_TARGETS_WITH_HEIGHT),
training_example_dict=TRAINING_EXAMPLE_DICT,
normalization_type_string=normalization.MINMAX_NORM_STRING,
min_normalized_value=MIN_NORMALIZED_VALUE,
max_normalized_value=MAX_NORMALIZED_VALUE,
separate_heights=True,
apply_to_predictors=False,
apply_to_vector_targets=True,
apply_to_scalar_targets=True)
self.assertTrue(
_compare_example_dicts(this_example_dict,
EXAMPLE_DICT_DENORM_TARGETS_WITH_HEIGHT))
def _run(model_file_name, example_dir_name, first_time_string,
last_time_string, exclude_summit_greenland, output_file_name):
"""Applies trained neural net in inference mode.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param example_dir_name: Same.
:param first_time_string: Same.
:param last_time_string: Same.
:param exclude_summit_greenland: Same.
:param output_file_name: Same.
"""
first_time_unix_sec = time_conversion.string_to_unix_sec(
first_time_string, TIME_FORMAT)
last_time_unix_sec = time_conversion.string_to_unix_sec(
last_time_string, TIME_FORMAT)
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = neural_net.read_model(model_file_name)
metafile_name = neural_net.find_metafile(
model_dir_name=os.path.split(model_file_name)[0],
raise_error_if_missing=True)
print('Reading metadata from: "{0:s}"...'.format(metafile_name))
metadata_dict = neural_net.read_metafile(metafile_name)
generator_option_dict = copy.deepcopy(
metadata_dict[neural_net.TRAINING_OPTIONS_KEY])
generator_option_dict[neural_net.EXAMPLE_DIRECTORY_KEY] = example_dir_name
generator_option_dict[neural_net.FIRST_TIME_KEY] = first_time_unix_sec
generator_option_dict[neural_net.LAST_TIME_KEY] = last_time_unix_sec
vector_target_norm_type_string = copy.deepcopy(
generator_option_dict[neural_net.VECTOR_TARGET_NORM_TYPE_KEY])
scalar_target_norm_type_string = copy.deepcopy(
generator_option_dict[neural_net.SCALAR_TARGET_NORM_TYPE_KEY])
generator_option_dict[neural_net.VECTOR_TARGET_NORM_TYPE_KEY] = None
generator_option_dict[neural_net.SCALAR_TARGET_NORM_TYPE_KEY] = None
net_type_string = metadata_dict[neural_net.NET_TYPE_KEY]
predictor_matrix, target_array, example_id_strings = neural_net.create_data(
option_dict=generator_option_dict,
for_inference=True,
net_type_string=net_type_string,
exclude_summit_greenland=exclude_summit_greenland)
print(SEPARATOR_STRING)
exec_start_time_unix_sec = time.time()
prediction_array = neural_net.apply_model(
model_object=model_object,
predictor_matrix=predictor_matrix,
num_examples_per_batch=NUM_EXAMPLES_PER_BATCH,
net_type_string=net_type_string,
verbose=True)
print(SEPARATOR_STRING)
print('Time to apply neural net = {0:.4f} seconds'.format(
time.time() - exec_start_time_unix_sec))
vector_target_matrix = target_array[0]
vector_prediction_matrix = prediction_array[0]
if len(target_array) == 2:
scalar_target_matrix = target_array[1]
scalar_prediction_matrix = prediction_array[1]
else:
scalar_target_matrix = None
scalar_prediction_matrix = None
target_example_dict = _targets_numpy_to_dict(
scalar_target_matrix=scalar_target_matrix,
vector_target_matrix=vector_target_matrix,
model_metadata_dict=metadata_dict)
prediction_example_dict = _targets_numpy_to_dict(
scalar_target_matrix=scalar_prediction_matrix,
vector_target_matrix=vector_prediction_matrix,
model_metadata_dict=metadata_dict)
normalization_file_name = (
generator_option_dict[neural_net.NORMALIZATION_FILE_KEY])
print(('Reading training examples (for normalization) from: "{0:s}"...'
).format(normalization_file_name))
training_example_dict = example_io.read_file(normalization_file_name)
training_example_dict = example_utils.subset_by_height(
example_dict=training_example_dict,
heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY])
num_examples = len(example_id_strings)
num_heights = len(prediction_example_dict[example_utils.HEIGHTS_KEY])
this_dict = {
example_utils.VECTOR_PREDICTOR_NAMES_KEY: [],
example_utils.VECTOR_PREDICTOR_VALS_KEY:
numpy.full((num_examples, num_heights, 0), 0.),
example_utils.SCALAR_PREDICTOR_NAMES_KEY: [],
example_utils.SCALAR_PREDICTOR_VALS_KEY:
numpy.full((num_examples, 0), 0.)
}
target_example_dict.update(this_dict)
prediction_example_dict.update(this_dict)
if vector_target_norm_type_string is not None:
print('Denormalizing predicted vectors...')
# down_flux_inc_matrix_w_m03 = example_utils.get_field_from_dict(
# example_dict=prediction_example_dict,
# field_name=example_utils.SHORTWAVE_DOWN_FLUX_INC_NAME
# )
# print(down_flux_inc_matrix_w_m03[0, ...])
# print('\n')
prediction_example_dict = normalization.denormalize_data(
new_example_dict=prediction_example_dict,
training_example_dict=training_example_dict,
normalization_type_string=vector_target_norm_type_string,
min_normalized_value=generator_option_dict[
neural_net.VECTOR_TARGET_MIN_VALUE_KEY],
max_normalized_value=generator_option_dict[
neural_net.VECTOR_TARGET_MAX_VALUE_KEY],
separate_heights=True,
apply_to_predictors=False,
apply_to_vector_targets=True,
apply_to_scalar_targets=False)
# down_flux_inc_matrix_w_m03 = example_utils.get_field_from_dict(
# example_dict=prediction_example_dict,
# field_name=example_utils.SHORTWAVE_DOWN_FLUX_INC_NAME
# )
# print(down_flux_inc_matrix_w_m03[0, ...])
# print('\n\n\n')
if scalar_target_norm_type_string is not None:
print('Denormalizing predicted scalars...')
prediction_example_dict = normalization.denormalize_data(
new_example_dict=prediction_example_dict,
training_example_dict=training_example_dict,
normalization_type_string=scalar_target_norm_type_string,
min_normalized_value=generator_option_dict[
neural_net.SCALAR_TARGET_MIN_VALUE_KEY],
max_normalized_value=generator_option_dict[
neural_net.SCALAR_TARGET_MAX_VALUE_KEY],
separate_heights=True,
apply_to_predictors=False,
apply_to_vector_targets=False,
apply_to_scalar_targets=True)
add_heating_rate = generator_option_dict[neural_net.OMIT_HEATING_RATE_KEY]
if add_heating_rate:
pressure_matrix_pascals = _get_unnormalized_pressure(
model_metadata_dict=metadata_dict,
example_id_strings=example_id_strings)
prediction_example_dict = _get_predicted_heating_rates(
prediction_example_dict=prediction_example_dict,
pressure_matrix_pascals=pressure_matrix_pascals,
model_metadata_dict=metadata_dict)
vector_target_names = (
generator_option_dict[neural_net.VECTOR_TARGET_NAMES_KEY])
if example_utils.SHORTWAVE_HEATING_RATE_NAME in vector_target_names:
heating_rate_index = vector_target_names.index(
example_utils.SHORTWAVE_HEATING_RATE_NAME)
heights_m_agl = generator_option_dict[neural_net.HEIGHTS_KEY]
height_indices = numpy.where(
heights_m_agl >= ZERO_HEATING_HEIGHT_M_AGL)[0]
vector_target_matrix = (
prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY])
vector_target_matrix[..., heating_rate_index][..., height_indices] = 0.
prediction_example_dict[example_utils.VECTOR_TARGET_VALS_KEY] = (
vector_target_matrix)
all_heights_m_agl = generator_option_dict[neural_net.HEIGHTS_KEY]
desired_heights_m_agl = (
all_heights_m_agl[all_heights_m_agl < MAX_HEIGHT_M_AGL])
target_example_dict = example_utils.subset_by_height(
example_dict=target_example_dict, heights_m_agl=desired_heights_m_agl)
prediction_example_dict = example_utils.subset_by_height(
example_dict=prediction_example_dict,
heights_m_agl=desired_heights_m_agl)
print('Writing target (actual) and predicted values to: "{0:s}"...'.format(
output_file_name))
prediction_io.write_file(netcdf_file_name=output_file_name,
scalar_target_matrix=target_example_dict[
example_utils.SCALAR_TARGET_VALS_KEY],
vector_target_matrix=target_example_dict[
example_utils.VECTOR_TARGET_VALS_KEY],
scalar_prediction_matrix=prediction_example_dict[
example_utils.SCALAR_TARGET_VALS_KEY],
vector_prediction_matrix=prediction_example_dict[
example_utils.VECTOR_TARGET_VALS_KEY],
heights_m_agl=desired_heights_m_agl,
example_id_strings=example_id_strings,
model_file_name=model_file_name)
def _run(model_file_name, example_file_name, num_examples, example_dir_name,
example_id_file_name, layer_name, neuron_indices, ideal_activation,
num_iterations, learning_rate, l2_weight, output_file_name):
"""Runs backwards optimization.
This is effectively the main method.
:param model_file_name: See documentation at top of file.
:param example_file_name: Same.
:param num_examples: Same.
:param example_dir_name: Same.
:param example_id_file_name: Same.
:param layer_name: Same.
:param neuron_indices: Same.
:param ideal_activation: Same.
:param num_iterations: Same.
:param learning_rate: Same.
:param l2_weight: Same.
:param output_file_name: Same.
"""
print('Reading model from: "{0:s}"...'.format(model_file_name))
model_object = neural_net.read_model(model_file_name)
metafile_name = neural_net.find_metafile(
model_dir_name=os.path.split(model_file_name)[0],
raise_error_if_missing=True
)
print('Reading metadata from: "{0:s}"...'.format(metafile_name))
metadata_dict = neural_net.read_metafile(metafile_name)
predictor_matrix, _, example_id_strings = (
misc_utils.get_examples_for_inference(
model_metadata_dict=metadata_dict,
example_file_name=example_file_name,
num_examples=num_examples, example_dir_name=example_dir_name,
example_id_file_name=example_id_file_name
)
)
print(SEPARATOR_STRING)
generator_option_dict = metadata_dict[neural_net.TRAINING_OPTIONS_KEY]
normalization_file_name = (
generator_option_dict[neural_net.NORMALIZATION_FILE_KEY]
)
print((
'Reading training examples (for normalization) from: "{0:s}"...'
).format(
normalization_file_name
))
training_example_dict = example_io.read_file(normalization_file_name)
training_example_dict = example_utils.subset_by_height(
example_dict=training_example_dict,
heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY]
)
num_examples = len(example_id_strings)
bwo_dict = None
for i in range(num_examples):
this_bwo_dict = bwo.optimize_input_for_neuron(
model_object=model_object,
init_function_or_matrix=predictor_matrix[i, ...],
layer_name=layer_name, neuron_indices=neuron_indices,
ideal_activation=ideal_activation, num_iterations=num_iterations,
learning_rate=learning_rate, l2_weight=l2_weight
)
if i == num_examples - 1:
print(SEPARATOR_STRING)
else:
print(MINOR_SEPARATOR_STRING)
if bwo_dict is None:
these_dim = numpy.array(
(num_examples,) +
this_bwo_dict[bwo.INITIAL_PREDICTORS_KEY].shape[1:],
dtype=int
)
bwo_dict = {
bwo.INITIAL_PREDICTORS_KEY: numpy.full(these_dim, numpy.nan),
bwo.FINAL_PREDICTORS_KEY: numpy.full(these_dim, numpy.nan),
bwo.INITIAL_ACTIVATIONS_KEY:
numpy.full(num_examples, numpy.nan),
bwo.FINAL_ACTIVATIONS_KEY: numpy.full(num_examples, numpy.nan)
}
bwo_dict[bwo.INITIAL_PREDICTORS_KEY][i, ...] = (
this_bwo_dict[bwo.INITIAL_PREDICTORS_KEY][0, ...]
)
bwo_dict[bwo.FINAL_PREDICTORS_KEY][i, ...] = (
this_bwo_dict[bwo.FINAL_PREDICTORS_KEY][0, ...]
)
bwo_dict[bwo.INITIAL_ACTIVATIONS_KEY][i] = (
this_bwo_dict[bwo.INITIAL_ACTIVATION_KEY]
)
bwo_dict[bwo.FINAL_ACTIVATIONS_KEY][i] = (
this_bwo_dict[bwo.FINAL_ACTIVATION_KEY]
)
if example_file_name == '':
example_file_name = example_io.find_many_files(
directory_name=example_dir_name,
first_time_unix_sec=0, last_time_unix_sec=int(1e12),
raise_error_if_any_missing=False, raise_error_if_all_missing=True
)[0]
first_example_dict = example_io.read_file(example_file_name)
first_example_dict = example_utils.subset_by_height(
example_dict=first_example_dict,
heights_m_agl=generator_option_dict[neural_net.HEIGHTS_KEY]
)
net_type_string = metadata_dict[neural_net.NET_TYPE_KEY]
init_example_dict = copy.deepcopy(first_example_dict)
this_example_dict = neural_net.predictors_numpy_to_dict(
predictor_matrix=bwo_dict[bwo.INITIAL_PREDICTORS_KEY],
example_dict=init_example_dict, net_type_string=net_type_string
)
init_example_dict.update(this_example_dict)
if generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY] is not None:
init_example_dict = normalization.denormalize_data(
new_example_dict=init_example_dict,
training_example_dict=training_example_dict,
normalization_type_string=
generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY],
min_normalized_value=
generator_option_dict[neural_net.PREDICTOR_MIN_NORM_VALUE_KEY],
max_normalized_value=
generator_option_dict[neural_net.PREDICTOR_MAX_NORM_VALUE_KEY],
separate_heights=True, apply_to_predictors=True,
apply_to_vector_targets=False, apply_to_scalar_targets=False
)
init_scalar_predictor_matrix = (
init_example_dict[example_utils.SCALAR_PREDICTOR_VALS_KEY]
)
init_vector_predictor_matrix = (
init_example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY]
)
final_example_dict = copy.deepcopy(first_example_dict)
this_example_dict = neural_net.predictors_numpy_to_dict(
predictor_matrix=bwo_dict[bwo.FINAL_PREDICTORS_KEY],
example_dict=final_example_dict, net_type_string=net_type_string
)
final_example_dict.update(this_example_dict)
if generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY] is not None:
final_example_dict = normalization.denormalize_data(
new_example_dict=final_example_dict,
training_example_dict=training_example_dict,
normalization_type_string=
generator_option_dict[neural_net.PREDICTOR_NORM_TYPE_KEY],
min_normalized_value=
generator_option_dict[neural_net.PREDICTOR_MIN_NORM_VALUE_KEY],
max_normalized_value=
generator_option_dict[neural_net.PREDICTOR_MAX_NORM_VALUE_KEY],
separate_heights=True, apply_to_predictors=True,
apply_to_vector_targets=False, apply_to_scalar_targets=False
)
final_scalar_predictor_matrix = (
final_example_dict[example_utils.SCALAR_PREDICTOR_VALS_KEY]
)
final_vector_predictor_matrix = (
final_example_dict[example_utils.VECTOR_PREDICTOR_VALS_KEY]
)
print('Writing results to file: "{0:s}"...'.format(output_file_name))
bwo.write_file(
netcdf_file_name=output_file_name,
init_scalar_predictor_matrix=init_scalar_predictor_matrix,
final_scalar_predictor_matrix=final_scalar_predictor_matrix,
init_vector_predictor_matrix=init_vector_predictor_matrix,
final_vector_predictor_matrix=final_vector_predictor_matrix,
initial_activations=bwo_dict[bwo.INITIAL_ACTIVATIONS_KEY],
final_activations=bwo_dict[bwo.FINAL_ACTIVATIONS_KEY],
example_id_strings=example_id_strings, model_file_name=model_file_name,
layer_name=layer_name, neuron_indices=neuron_indices,
ideal_activation=ideal_activation, num_iterations=num_iterations,
learning_rate=learning_rate, l2_weight=l2_weight
)
