def test_permute_one_predictor_third(self):
"""Ensures correct output from permute_one_predictor.
In this case, using third set of inputs.
"""
these_new_matrices, these_permuted_values = (
permutation_utils.permute_one_predictor(
predictor_matrices=copy.deepcopy(
PREDICTOR_MATRICES_FOR_PERMUTN),
separate_heights=THIRD_SEPARATE_FLAG_FOR_PERMUTN,
matrix_index=THIRD_MATRIX_INDEX_FOR_PERMUTN,
predictor_index=THIRD_PRED_INDEX_FOR_PERMUTN,
permuted_values=None))
self.assertTrue(
_compare_before_and_after_permutn(
orig_predictor_matrices=PREDICTOR_MATRICES_FOR_PERMUTN,
new_predictor_matrices=these_new_matrices,
separate_heights=THIRD_SEPARATE_FLAG_FOR_PERMUTN,
matrix_index=THIRD_MATRIX_INDEX_FOR_PERMUTN,
predictor_index=THIRD_PRED_INDEX_FOR_PERMUTN))
these_newnew_matrices = permutation_utils.permute_one_predictor(
predictor_matrices=copy.deepcopy(these_new_matrices),
separate_heights=THIRD_SEPARATE_FLAG_FOR_PERMUTN,
matrix_index=THIRD_MATRIX_INDEX_FOR_PERMUTN,
predictor_index=THIRD_PRED_INDEX_FOR_PERMUTN,
permuted_values=these_permuted_values)[0]
for i in range(len(these_new_matrices)):
self.assertTrue(
numpy.allclose(these_newnew_matrices[i],
these_new_matrices[i],
atol=TOLERANCE))
def test_unpermute_one_predictor_second(self):
"""Ensures correct output from _unpermute_one_predictor.
In this case, using second set of inputs.
"""
these_new_matrices = permutation_utils.permute_one_predictor(
predictor_matrices=copy.deepcopy(PREDICTOR_MATRICES_FOR_PERMUTN),
separate_heights=SECOND_SEPARATE_FLAG_FOR_PERMUTN,
matrix_index=SECOND_MATRIX_INDEX_FOR_PERMUTN,
predictor_index=SECOND_PRED_INDEX_FOR_PERMUTN,
permuted_values=None)[0]
these_orig_matrices = permutation_utils._unpermute_one_predictor(
predictor_matrices=copy.deepcopy(these_new_matrices),
clean_predictor_matrices=PREDICTOR_MATRICES_FOR_PERMUTN,
separate_heights=SECOND_SEPARATE_FLAG_FOR_PERMUTN,
matrix_index=SECOND_MATRIX_INDEX_FOR_PERMUTN,
predictor_index=SECOND_PRED_INDEX_FOR_PERMUTN)
for i in range(len(these_new_matrices)):
self.assertTrue(
numpy.allclose(these_orig_matrices[i],
PREDICTOR_MATRICES_FOR_PERMUTN[i],
atol=TOLERANCE))
def run_backwards_test(
model_object, predictor_matrices, target_values, cnn_metadata_dict,
cost_function, separate_radar_heights=False,
num_bootstrap_reps=DEFAULT_NUM_BOOTSTRAP_REPS):
"""Runs backwards permutation test.
P = number of predictors to unpermute
B = number of bootstrap replicates
:param model_object: See doc for `run_forward_test`.
:param predictor_matrices: Same.
:param target_values: Same.
:param cnn_metadata_dict: Same.
:param cost_function: Same.
:param separate_radar_heights: Same.
:param num_bootstrap_reps: Same.
:return: result_dict: Dictionary with the following keys.
result_dict["best_predictor_names"]: length-P list of best
predictors. The [j]th element is the name of the [j]th predictor to be
permanently unpermuted.
result_dict["best_cost_matrix"]: P-by-B numpy array of costs after
unpermutation.
result_dict["original_cost_array"]: length-B numpy array of costs
before unpermutation.
result_dict["step1_predictor_names"]: length-P list of predictors in
the order that they were unpermuted in step 1.
result_dict["step1_cost_matrix"]: P-by-B numpy array of costs after
unpermutation in step 1.
result_dict["backwards_test"]: Boolean flag (always True).
"""
# Deal with input args.
error_checking.assert_is_integer_numpy_array(target_values)
error_checking.assert_is_geq_numpy_array(target_values, 0)
error_checking.assert_is_integer(num_bootstrap_reps)
num_bootstrap_reps = max([num_bootstrap_reps, 1])
if cnn_metadata_dict[cnn.CONV_2D3D_KEY]:
prediction_function = prediction_function_2d3d_cnn
else:
num_radar_dimensions = len(predictor_matrices[0].shape) - 2
if num_radar_dimensions == 2:
prediction_function = prediction_function_2d_cnn
else:
prediction_function = prediction_function_3d_cnn
predictor_names_by_matrix = create_nice_predictor_names(
predictor_matrices=predictor_matrices,
cnn_metadata_dict=cnn_metadata_dict,
separate_radar_heights=separate_radar_heights)
num_matrices = len(predictor_names_by_matrix)
for i in range(num_matrices):
print('Predictors in {0:d}th matrix:\n{1:s}\n'.format(
i + 1, str(predictor_names_by_matrix[i])
))
print(SEPARATOR_STRING)
# Permute all predictors.
num_matrices = len(predictor_matrices)
clean_predictor_matrices = copy.deepcopy(predictor_matrices)
for i in range(num_matrices):
this_num_predictors = len(predictor_names_by_matrix[i])
for j in range(this_num_predictors):
predictor_matrices = permutation_utils.permute_one_predictor(
predictor_matrices=predictor_matrices,
separate_heights=separate_radar_heights,
matrix_index=i, predictor_index=j
)[0]
# Find original cost (before unpermutation).
print('Finding original cost (before unpermutation)...')
class_probability_matrix = prediction_function(
model_object, predictor_matrices)
print(MINOR_SEPARATOR_STRING)
original_cost_array = permutation_utils.bootstrap_cost(
target_values=target_values,
class_probability_matrix=class_probability_matrix,
cost_function=cost_function, num_replicates=num_bootstrap_reps)
# Do the dirty work.
permuted_flags_by_matrix = [
numpy.full(len(n), 1, dtype=bool)
for n in predictor_names_by_matrix
]
step_num = 0
step1_predictor_names = None
step1_cost_matrix = None
best_predictor_names = []
best_cost_matrix = numpy.full((0, num_bootstrap_reps), numpy.nan)
while True:
print(MINOR_SEPARATOR_STRING)
step_num += 1
this_dict = permutation_utils.run_backwards_test_one_step(
model_object=model_object, predictor_matrices=predictor_matrices,
clean_predictor_matrices=clean_predictor_matrices,
predictor_names_by_matrix=predictor_names_by_matrix,
target_values=target_values,
prediction_function=prediction_function,
cost_function=cost_function,
separate_heights=separate_radar_heights,
num_bootstrap_reps=num_bootstrap_reps, step_num=step_num,
permuted_flags_by_matrix=permuted_flags_by_matrix)
if this_dict is None:
break
predictor_matrices = this_dict[PREDICTOR_MATRICES_KEY]
permuted_flags_by_matrix = this_dict[PERMUTED_FLAGS_KEY]
best_predictor_names.append(this_dict[BEST_PREDICTOR_KEY])
this_best_cost_array = this_dict[BEST_COST_ARRAY_KEY]
this_best_cost_matrix = numpy.reshape(
this_best_cost_array, (1, len(this_best_cost_array))
)
best_cost_matrix = numpy.concatenate(
(best_cost_matrix, this_best_cost_matrix), axis=0
)
if step_num == 1:
step1_predictor_names = this_dict[UNPERMUTED_PREDICTORS_KEY]
step1_cost_matrix = this_dict[UNPERMUTED_COST_MATRIX_KEY]
return {
permutation_utils.BEST_PREDICTORS_KEY: best_predictor_names,
permutation_utils.BEST_COST_MATRIX_KEY: best_cost_matrix,
permutation_utils.ORIGINAL_COST_ARRAY_KEY: original_cost_array,
permutation_utils.STEP1_PREDICTORS_KEY: step1_predictor_names,
permutation_utils.STEP1_COST_MATRIX_KEY: step1_cost_matrix,
permutation_utils.BACKWARDS_FLAG: True
}