def prediction_function(predictor_matrix):
"""Prediction function itself.
:param predictor_matrix: See doc for `run_forward_test_one_step`.
:return: prediction_matrices: 1-D list of numpy arrays, each containing
predicted values.
"""
net_type_string = model_metadata_dict[neural_net.NET_TYPE_KEY]
if net_type_string == neural_net.DENSE_NET_TYPE_STRING:
generator_option_dict = (
model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY])
example_dict = {
example_utils.SCALAR_PREDICTOR_NAMES_KEY:
generator_option_dict[neural_net.SCALAR_PREDICTOR_NAMES_KEY],
example_utils.VECTOR_PREDICTOR_NAMES_KEY:
generator_option_dict[neural_net.VECTOR_PREDICTOR_NAMES_KEY],
example_utils.HEIGHTS_KEY:
generator_option_dict[neural_net.HEIGHTS_KEY]
}
new_example_dict = neural_net.predictors_numpy_to_dict(
predictor_matrix=predictor_matrix,
example_dict=example_dict,
net_type_string=neural_net.CNN_TYPE_STRING)
example_dict.update(new_example_dict)
this_predictor_matrix = neural_net.predictors_dict_to_numpy(
example_dict=example_dict, net_type_string=net_type_string)[0]
else:
this_predictor_matrix = predictor_matrix
return neural_net.apply_model(model_object=model_object,
predictor_matrix=this_predictor_matrix,
num_examples_per_batch=1000,
net_type_string=net_type_string,
is_loss_constrained_mse=False,
verbose=False)
def test_predictors_dict_to_numpy_u_net(self):
"""Ensures correct output from predictors_dict_to_numpy.
In this case, neural-net type is U-net.
"""
(this_predictor_matrix, this_predictor_name_matrix,
this_height_matrix_m_agl) = neural_net.predictors_dict_to_numpy(
example_dict=EXAMPLE_DICT,
net_type_string=neural_net.U_NET_TYPE_STRING)
self.assertTrue(
numpy.allclose(this_predictor_matrix,
U_NET_PREDICTOR_MATRIX,
atol=TOLERANCE))
self.assertTrue(
numpy.array_equal(this_predictor_name_matrix,
U_NET_PREDICTOR_NAME_MATRIX))
self.assertTrue(
numpy.allclose(this_height_matrix_m_agl,
U_NET_HEIGHT_MATRIX_M_AGL,
atol=TOLERANCE,
equal_nan=True))
def run_backwards_test(predictor_matrix,
target_matrices,
model_object,
model_metadata_dict,
cost_function,
shuffle_profiles_together=True,
num_bootstrap_reps=DEFAULT_NUM_BOOTSTRAP_REPS):
"""Runs backwards version of permutation test (both single- and multi-pass).
B = number of bootstrap replicates
N = number of predictors (either channels or height/channel pairs) available
to permute
:param predictor_matrix: See doc for `run_forward_test`.
:param target_matrices: Same.
:param model_object: Same.
:param model_metadata_dict: Same.
:param cost_function: Same.
:param shuffle_profiles_together: Same.
:param num_bootstrap_reps: Same.
:return: result_dict: Dictionary with the following keys.
result_dict['orig_cost_estimates']: length-B numpy array with estimates of
original cost (before *de*permutation).
result_dict['best_predictor_names']: length-N list with best predictor at
each step.
result_dict['best_heights_m_agl']: length-N numpy array of corresponding
heights (metres above ground level). This may be None.
result_dict['best_cost_matrix']: N-by-B numpy array of costs after
*de*permutation at each step.
result_dict['step1_predictor_names']: length-N list with predictors in order
that they were *de*permuted in step 1.
result_dict['step1_heights_m_agl']: length-N numpy array of corresponding
heights (metres above ground level). This may be None.
result_dict['step1_cost_matrix']: N-by-B numpy array of costs after
*de*permutation in step 1.
result_dict['is_backwards_test']: Boolean flag (always True for this
method).
"""
error_checking.assert_is_boolean(shuffle_profiles_together)
error_checking.assert_is_integer(num_bootstrap_reps)
num_bootstrap_reps = numpy.maximum(num_bootstrap_reps, 1)
generator_option_dict = model_metadata_dict[
neural_net.TRAINING_OPTIONS_KEY]
example_dict = {
example_utils.SCALAR_PREDICTOR_NAMES_KEY:
generator_option_dict[neural_net.SCALAR_PREDICTOR_NAMES_KEY],
example_utils.VECTOR_PREDICTOR_NAMES_KEY:
generator_option_dict[neural_net.VECTOR_PREDICTOR_NAMES_KEY],
example_utils.HEIGHTS_KEY:
generator_option_dict[neural_net.HEIGHTS_KEY]
}
new_example_dict = neural_net.predictors_numpy_to_dict(
predictor_matrix=predictor_matrix,
example_dict=example_dict,
net_type_string=model_metadata_dict[neural_net.NET_TYPE_KEY])
example_dict.update(new_example_dict)
predictor_matrix, predictor_name_matrix, height_matrix_m_agl = (
neural_net.predictors_dict_to_numpy(
example_dict=example_dict,
net_type_string=neural_net.CNN_TYPE_STRING))
scalar_channel_flags = numpy.isnan(height_matrix_m_agl[0, :])
prediction_function = _make_prediction_function(
model_object=model_object, model_metadata_dict=model_metadata_dict)
# Permute all predictors.
num_channels = predictor_matrix.shape[-1]
if shuffle_profiles_together:
num_heights = 1
else:
num_heights = predictor_matrix.shape[-2]
clean_predictor_matrix = predictor_matrix + 0.
for j in range(num_heights):
for k in range(num_channels):
predictor_matrix = _permute_values(
predictor_matrix=predictor_matrix,
channel_index=k,
height_index=None if shuffle_profiles_together else j)[0]
# Find original cost (before *de*permutation).
print('Finding original cost (before *de*permutation)...')
orig_cost_estimates = _bootstrap_cost(
target_matrices=target_matrices,
prediction_matrices=prediction_function(predictor_matrix),
cost_function=cost_function,
num_replicates=num_bootstrap_reps)
# Do dirty work.
permuted_flag_matrix = numpy.full(predictor_matrix.shape[1:],
True,
dtype=bool)
best_predictor_names = []
best_cost_matrix = numpy.full((0, num_bootstrap_reps), numpy.nan)
if shuffle_profiles_together:
best_heights_m_agl = None
else:
best_heights_m_agl = []
step1_predictor_names = None
step1_heights_m_agl = None
step1_cost_matrix = None
step_num = 0
while True:
print(MINOR_SEPARATOR_STRING)
step_num += 1
this_result_dict = run_backwards_test_one_step(
predictor_matrix=predictor_matrix,
clean_predictor_matrix=clean_predictor_matrix,
target_matrices=target_matrices,
prediction_function=prediction_function,
cost_function=cost_function,
permuted_flag_matrix=permuted_flag_matrix,
scalar_channel_flags=scalar_channel_flags,
shuffle_profiles_together=shuffle_profiles_together,
num_bootstrap_reps=num_bootstrap_reps)
if this_result_dict is None:
break
predictor_matrix = this_result_dict[PREDICTORS_KEY]
permuted_flag_matrix = this_result_dict[PERMUTED_FLAGS_KEY]
these_predictor_names, these_heights_m_agl = (
_predictor_indices_to_metadata(
all_predictor_name_matrix=predictor_name_matrix,
all_height_matrix_m_agl=height_matrix_m_agl,
one_step_result_dict=this_result_dict))
this_best_index = numpy.argmin(
numpy.mean(this_result_dict[DEPERMUTED_COSTS_KEY], axis=1))
best_predictor_names.append(these_predictor_names[this_best_index])
best_cost_matrix = numpy.concatenate(
(best_cost_matrix,
this_result_dict[DEPERMUTED_COSTS_KEY][[this_best_index], :]),
axis=0)
log_string = 'Best predictor at {0:d}th step = {1:s}'.format(
step_num, best_predictor_names[-1])
if not shuffle_profiles_together:
best_heights_m_agl.append(these_heights_m_agl[this_best_index])
log_string += ' at {0:s} m AGL'.format(best_heights_m_agl[-1])
log_string += ' ... cost = {0:f}'.format(
numpy.mean(best_cost_matrix[-1, :]))
print(log_string)
if step_num != 1:
continue
step1_predictor_names = copy.deepcopy(these_predictor_names)
step1_cost_matrix = this_result_dict[DEPERMUTED_COSTS_KEY] + 0.
if not shuffle_profiles_together:
step1_heights_m_agl = these_heights_m_agl + 0
return {
ORIGINAL_COST_KEY: orig_cost_estimates,
BEST_PREDICTORS_KEY: best_predictor_names,
BEST_HEIGHTS_KEY: best_heights_m_agl,
BEST_COSTS_KEY: best_cost_matrix,
STEP1_PREDICTORS_KEY: step1_predictor_names,
STEP1_HEIGHTS_KEY: step1_heights_m_agl,
STEP1_COSTS_KEY: step1_cost_matrix,
BACKWARDS_FLAG_KEY: True
}