def prediction_function(predictor_matrix): """Prediction function itself. :param predictor_matrix: See doc for `run_forward_test_one_step`. :return: prediction_matrices: 1-D list of numpy arrays, each containing predicted values. """ net_type_string = model_metadata_dict[neural_net.NET_TYPE_KEY] if net_type_string == neural_net.DENSE_NET_TYPE_STRING: generator_option_dict = ( model_metadata_dict[neural_net.TRAINING_OPTIONS_KEY]) example_dict = { example_utils.SCALAR_PREDICTOR_NAMES_KEY: generator_option_dict[neural_net.SCALAR_PREDICTOR_NAMES_KEY], example_utils.VECTOR_PREDICTOR_NAMES_KEY: generator_option_dict[neural_net.VECTOR_PREDICTOR_NAMES_KEY], example_utils.HEIGHTS_KEY: generator_option_dict[neural_net.HEIGHTS_KEY] } new_example_dict = neural_net.predictors_numpy_to_dict( predictor_matrix=predictor_matrix, example_dict=example_dict, net_type_string=neural_net.CNN_TYPE_STRING) example_dict.update(new_example_dict) this_predictor_matrix = neural_net.predictors_dict_to_numpy( example_dict=example_dict, net_type_string=net_type_string)[0] else: this_predictor_matrix = predictor_matrix return neural_net.apply_model(model_object=model_object, predictor_matrix=this_predictor_matrix, num_examples_per_batch=1000, net_type_string=net_type_string, is_loss_constrained_mse=False, verbose=False)
def test_predictors_dict_to_numpy_u_net(self): """Ensures correct output from predictors_dict_to_numpy. In this case, neural-net type is U-net. """ (this_predictor_matrix, this_predictor_name_matrix, this_height_matrix_m_agl) = neural_net.predictors_dict_to_numpy( example_dict=EXAMPLE_DICT, net_type_string=neural_net.U_NET_TYPE_STRING) self.assertTrue( numpy.allclose(this_predictor_matrix, U_NET_PREDICTOR_MATRIX, atol=TOLERANCE)) self.assertTrue( numpy.array_equal(this_predictor_name_matrix, U_NET_PREDICTOR_NAME_MATRIX)) self.assertTrue( numpy.allclose(this_height_matrix_m_agl, U_NET_HEIGHT_MATRIX_M_AGL, atol=TOLERANCE, equal_nan=True))
def run_backwards_test(predictor_matrix, target_matrices, model_object, model_metadata_dict, cost_function, shuffle_profiles_together=True, num_bootstrap_reps=DEFAULT_NUM_BOOTSTRAP_REPS): """Runs backwards version of permutation test (both single- and multi-pass). B = number of bootstrap replicates N = number of predictors (either channels or height/channel pairs) available to permute :param predictor_matrix: See doc for `run_forward_test`. :param target_matrices: Same. :param model_object: Same. :param model_metadata_dict: Same. :param cost_function: Same. :param shuffle_profiles_together: Same. :param num_bootstrap_reps: Same. :return: result_dict: Dictionary with the following keys. result_dict['orig_cost_estimates']: length-B numpy array with estimates of original cost (before *de*permutation). result_dict['best_predictor_names']: length-N list with best predictor at each step. result_dict['best_heights_m_agl']: length-N numpy array of corresponding heights (metres above ground level). This may be None. result_dict['best_cost_matrix']: N-by-B numpy array of costs after *de*permutation at each step. result_dict['step1_predictor_names']: length-N list with predictors in order that they were *de*permuted in step 1. result_dict['step1_heights_m_agl']: length-N numpy array of corresponding heights (metres above ground level). This may be None. result_dict['step1_cost_matrix']: N-by-B numpy array of costs after *de*permutation in step 1. result_dict['is_backwards_test']: Boolean flag (always True for this method). """ error_checking.assert_is_boolean(shuffle_profiles_together) error_checking.assert_is_integer(num_bootstrap_reps) num_bootstrap_reps = numpy.maximum(num_bootstrap_reps, 1) generator_option_dict = model_metadata_dict[ neural_net.TRAINING_OPTIONS_KEY] example_dict = { example_utils.SCALAR_PREDICTOR_NAMES_KEY: generator_option_dict[neural_net.SCALAR_PREDICTOR_NAMES_KEY], example_utils.VECTOR_PREDICTOR_NAMES_KEY: generator_option_dict[neural_net.VECTOR_PREDICTOR_NAMES_KEY], example_utils.HEIGHTS_KEY: generator_option_dict[neural_net.HEIGHTS_KEY] } new_example_dict = neural_net.predictors_numpy_to_dict( predictor_matrix=predictor_matrix, example_dict=example_dict, net_type_string=model_metadata_dict[neural_net.NET_TYPE_KEY]) example_dict.update(new_example_dict) predictor_matrix, predictor_name_matrix, height_matrix_m_agl = ( neural_net.predictors_dict_to_numpy( example_dict=example_dict, net_type_string=neural_net.CNN_TYPE_STRING)) scalar_channel_flags = numpy.isnan(height_matrix_m_agl[0, :]) prediction_function = _make_prediction_function( model_object=model_object, model_metadata_dict=model_metadata_dict) # Permute all predictors. num_channels = predictor_matrix.shape[-1] if shuffle_profiles_together: num_heights = 1 else: num_heights = predictor_matrix.shape[-2] clean_predictor_matrix = predictor_matrix + 0. for j in range(num_heights): for k in range(num_channels): predictor_matrix = _permute_values( predictor_matrix=predictor_matrix, channel_index=k, height_index=None if shuffle_profiles_together else j)[0] # Find original cost (before *de*permutation). print('Finding original cost (before *de*permutation)...') orig_cost_estimates = _bootstrap_cost( target_matrices=target_matrices, prediction_matrices=prediction_function(predictor_matrix), cost_function=cost_function, num_replicates=num_bootstrap_reps) # Do dirty work. permuted_flag_matrix = numpy.full(predictor_matrix.shape[1:], True, dtype=bool) best_predictor_names = [] best_cost_matrix = numpy.full((0, num_bootstrap_reps), numpy.nan) if shuffle_profiles_together: best_heights_m_agl = None else: best_heights_m_agl = [] step1_predictor_names = None step1_heights_m_agl = None step1_cost_matrix = None step_num = 0 while True: print(MINOR_SEPARATOR_STRING) step_num += 1 this_result_dict = run_backwards_test_one_step( predictor_matrix=predictor_matrix, clean_predictor_matrix=clean_predictor_matrix, target_matrices=target_matrices, prediction_function=prediction_function, cost_function=cost_function, permuted_flag_matrix=permuted_flag_matrix, scalar_channel_flags=scalar_channel_flags, shuffle_profiles_together=shuffle_profiles_together, num_bootstrap_reps=num_bootstrap_reps) if this_result_dict is None: break predictor_matrix = this_result_dict[PREDICTORS_KEY] permuted_flag_matrix = this_result_dict[PERMUTED_FLAGS_KEY] these_predictor_names, these_heights_m_agl = ( _predictor_indices_to_metadata( all_predictor_name_matrix=predictor_name_matrix, all_height_matrix_m_agl=height_matrix_m_agl, one_step_result_dict=this_result_dict)) this_best_index = numpy.argmin( numpy.mean(this_result_dict[DEPERMUTED_COSTS_KEY], axis=1)) best_predictor_names.append(these_predictor_names[this_best_index]) best_cost_matrix = numpy.concatenate( (best_cost_matrix, this_result_dict[DEPERMUTED_COSTS_KEY][[this_best_index], :]), axis=0) log_string = 'Best predictor at {0:d}th step = {1:s}'.format( step_num, best_predictor_names[-1]) if not shuffle_profiles_together: best_heights_m_agl.append(these_heights_m_agl[this_best_index]) log_string += ' at {0:s} m AGL'.format(best_heights_m_agl[-1]) log_string += ' ... cost = {0:f}'.format( numpy.mean(best_cost_matrix[-1, :])) print(log_string) if step_num != 1: continue step1_predictor_names = copy.deepcopy(these_predictor_names) step1_cost_matrix = this_result_dict[DEPERMUTED_COSTS_KEY] + 0. if not shuffle_profiles_together: step1_heights_m_agl = these_heights_m_agl + 0 return { ORIGINAL_COST_KEY: orig_cost_estimates, BEST_PREDICTORS_KEY: best_predictor_names, BEST_HEIGHTS_KEY: best_heights_m_agl, BEST_COSTS_KEY: best_cost_matrix, STEP1_PREDICTORS_KEY: step1_predictor_names, STEP1_HEIGHTS_KEY: step1_heights_m_agl, STEP1_COSTS_KEY: step1_cost_matrix, BACKWARDS_FLAG_KEY: True }