def test_sample_by_class_wind(self): """Ensures correct output from sample_by_class. In this case, target variable = wind-speed category. """ these_indices = dl_utils.sample_by_class( sampling_fraction_by_class_dict= SAMPLING_FRACTION_BY_WIND_4CLASS_DICT, target_name=WIND_TARGET_NAME_4CLASSES, target_values=WIND_LABELS_TO_SAMPLE, num_examples_total=NUM_EXAMPLES_TOTAL, test_mode=True) self.assertTrue(numpy.array_equal(these_indices, WIND_INDICES_TO_KEEP))
def test_sample_by_class_tornado(self): """Ensures correct output from sample_by_class. In this case, target variable = tornado occurrence. """ these_indices = dl_utils.sample_by_class( sampling_fraction_by_class_dict= BALANCED_FRACTION_BY_TORNADO_CLASS_DICT, target_name=TORNADO_TARGET_NAME, target_values=TORNADO_LABELS_TO_SAMPLE, num_examples_total=NUM_EXAMPLES_TOTAL, test_mode=True) self.assertTrue( numpy.array_equal(these_indices, TORNADO_INDICES_TO_KEEP))
def _downsampling_base(storm_ids, storm_times_unix_sec, target_values, target_name, class_fraction_dict, test_mode=False): """Base for `downsample_for_training` and `downsample_for_non_training`. The procedure is described below. [1] Find all storm objects in the highest class (e.g., tornadic). Call this set {s_highest}. [2] Find all storm cells with at least one object in {s_highest}. Call this set {S_highest}. [3] Find all time steps with at least one storm cell in {S_highest}. Call this set {t_highest}. [4] Randomly remove a large fraction of time steps NOT in {t_highest}. [5] Downsample remaining storm objects, leaving a prescribed fraction in each class (according to `class_fraction_dict`). N = number of storm objects before downsampling K = number of storm objects after intermediate downsampling n = number of storm objects after final downsampling :param storm_ids: length-N list of storm IDs (strings). :param storm_times_unix_sec: length-N numpy array of corresponding times. :param target_values: length-N numpy array of corresponding target values (integer class labels). :param target_name: Name of target variable (must be accepted by `target_val_utils.target_name_to_params`). :param class_fraction_dict: Dictionary, where each key is an integer class label (-2 for "dead storm") and the corresponding value is the sampling fraction. :param test_mode: Never mind. Just leave this alone. :return: storm_ids: length-K list of storm IDs (strings). :return: storm_times_unix_sec: length-K numpy array of corresponding times. :return: target_values: length-K numpy array of corresponding target values. :return: indices_to_keep: length-n numpy array of indices to keep. These are indices are into the output arrays `storm_ids`, `storm_times_unix_sec`, and `target_values`. """ _report_class_fractions(target_values) error_checking.assert_is_boolean(test_mode) num_storm_objects = len(storm_ids) num_classes = target_val_utils.target_name_to_num_classes( target_name=target_name, include_dead_storms=False) # Step 1. print( 'Finding storm objects in class {0:d} (the highest class), yielding set' ' {{s_highest}}...').format(num_classes - 1) highest_class_indices = numpy.where(target_values == num_classes - 1)[0] print '{{s_highest}} contains {0:d} of {1:d} storm objects.'.format( len(highest_class_indices), num_storm_objects) # Step 2. print( 'Finding storm cells with at least one object in {s_highest}, ' 'yielding set {S_highest}...') highest_class_indices = _find_storm_cells( storm_id_by_object=storm_ids, desired_storm_cell_ids=[storm_ids[k] for k in highest_class_indices]) print '{{S_highest}} contains {0:d} of {1:d} storm objects.'.format( len(highest_class_indices), num_storm_objects) # Step 3. print( 'Finding all time steps with at least one storm cell in ' '{S_highest}, yielding set {t_highest}...') lower_class_times_unix_sec = ( set(storm_times_unix_sec.tolist()) - set(storm_times_unix_sec[highest_class_indices].tolist())) lower_class_times_unix_sec = numpy.array(list(lower_class_times_unix_sec), dtype=int) # Step 4. print 'Randomly removing {0:.1f}% of times not in {{t_highest}}...'.format( FRACTION_UNINTERESTING_TIMES_TO_OMIT * 100) this_num_times = int( numpy.round(FRACTION_UNINTERESTING_TIMES_TO_OMIT * len(lower_class_times_unix_sec))) if test_mode: times_to_remove_unix_sec = lower_class_times_unix_sec[:this_num_times] else: times_to_remove_unix_sec = numpy.random.choice( lower_class_times_unix_sec, size=this_num_times, replace=False) indices_to_keep = _find_uncovered_times( all_times_unix_sec=storm_times_unix_sec, covered_times_unix_sec=times_to_remove_unix_sec) storm_ids = [storm_ids[k] for k in indices_to_keep] storm_times_unix_sec = storm_times_unix_sec[indices_to_keep] target_values = target_values[indices_to_keep] _report_class_fractions(target_values) # Step 5. print 'Downsampling storm objects from remaining times...' indices_to_keep = dl_utils.sample_by_class( sampling_fraction_by_class_dict=class_fraction_dict, target_name=target_name, target_values=target_values, num_examples_total=LARGE_INTEGER, test_mode=test_mode) return storm_ids, storm_times_unix_sec, target_values, indices_to_keep
def _compute_scores(forecast_probabilities, observed_labels, num_bootstrap_reps, output_file_name, best_prob_threshold=None, downsampling_dict=None): """Computes evaluation scores. E = number of examples (storm objects) :param forecast_probabilities: length-E numpy array of forecast event probabilities. :param observed_labels: length-E numpy array of observations (1 for event, 0 for non-event). :param num_bootstrap_reps: Number of bootstrap replicates. :param output_file_name: Path to output file (will be written by `model_evaluation.write_evaluation`). :param best_prob_threshold: Best probability threshold. If None, will be determined on the fly. :param downsampling_dict: Dictionary with downsampling fractions. See doc for `deep_learning_utils.sample_by_class`. If this is None, downsampling will not be used. """ num_examples = len(observed_labels) num_examples_by_class = numpy.unique(observed_labels, return_counts=True)[-1] print('Number of examples by class (no downsampling): {0:s}'.format( str(num_examples_by_class))) positive_example_indices = numpy.where(observed_labels == 1)[0] negative_example_indices = numpy.where(observed_labels == 0)[0] if downsampling_dict is None: these_indices = numpy.linspace(0, num_examples - 1, num=num_examples, dtype=int) else: these_indices = dl_utils.sample_by_class( sampling_fraction_by_class_dict=downsampling_dict, target_name=DUMMY_TARGET_NAME, target_values=observed_labels, num_examples_total=num_examples) this_num_ex_by_class = numpy.unique(observed_labels[these_indices], return_counts=True)[-1] print('Number of examples by class (after downsampling): {0:s}'.format( str(this_num_ex_by_class))) all_prob_thresholds = model_eval.get_binarization_thresholds( threshold_arg=model_eval.THRESHOLD_ARG_FOR_UNIQUE_FORECASTS, forecast_probabilities=forecast_probabilities[these_indices], forecast_precision=FORECAST_PRECISION) if best_prob_threshold is None: best_prob_threshold, best_csi = ( model_eval.find_best_binarization_threshold( forecast_probabilities=forecast_probabilities[these_indices], observed_labels=observed_labels[these_indices], threshold_arg=all_prob_thresholds, criterion_function=model_eval.get_csi, optimization_direction=model_eval.MAX_OPTIMIZATION_STRING)) else: these_forecast_labels = model_eval.binarize_forecast_probs( forecast_probabilities=forecast_probabilities[these_indices], binarization_threshold=best_prob_threshold) this_contingency_dict = model_eval.get_contingency_table( forecast_labels=these_forecast_labels, observed_labels=observed_labels[these_indices]) best_csi = model_eval.get_csi(this_contingency_dict) print( ('Best probability threshold = {0:.4f} ... corresponding CSI = {1:.4f}' ).format(best_prob_threshold, best_csi)) num_examples_by_forecast_bin = model_eval.get_points_in_reliability_curve( forecast_probabilities=forecast_probabilities[these_indices], observed_labels=observed_labels[these_indices], num_forecast_bins=model_eval.DEFAULT_NUM_RELIABILITY_BINS)[-1] list_of_evaluation_tables = [] for i in range(num_bootstrap_reps): print(('Computing scores for {0:d}th of {1:d} bootstrap replicates...' ).format(i + 1, num_bootstrap_reps)) if num_bootstrap_reps == 1: if downsampling_dict is None: these_indices = numpy.linspace(0, num_examples - 1, num=num_examples, dtype=int) else: these_indices = dl_utils.sample_by_class( sampling_fraction_by_class_dict=downsampling_dict, target_name=DUMMY_TARGET_NAME, target_values=observed_labels, num_examples_total=num_examples) else: if len(positive_example_indices) > 0: these_positive_indices = bootstrapping.draw_sample( positive_example_indices)[0] else: these_positive_indices = numpy.array([], dtype=int) these_negative_indices = bootstrapping.draw_sample( negative_example_indices)[0] these_indices = numpy.concatenate( (these_positive_indices, these_negative_indices)) if downsampling_dict is not None: these_subindices = dl_utils.sample_by_class( sampling_fraction_by_class_dict=downsampling_dict, target_name=DUMMY_TARGET_NAME, target_values=observed_labels[these_indices], num_examples_total=num_examples) these_indices = these_indices[these_subindices] if downsampling_dict is not None: this_num_ex_by_class = numpy.unique(observed_labels[these_indices], return_counts=True)[-1] print('Number of examples by class: {0:s}'.format( str(this_num_ex_by_class))) this_evaluation_table = model_eval.run_evaluation( forecast_probabilities=forecast_probabilities[these_indices], observed_labels=observed_labels[these_indices], best_prob_threshold=best_prob_threshold, all_prob_thresholds=all_prob_thresholds, climatology=numpy.mean(observed_labels[these_indices])) list_of_evaluation_tables.append(this_evaluation_table) if i == num_bootstrap_reps - 1: print(SEPARATOR_STRING) else: print(MINOR_SEPARATOR_STRING) if i == 0: continue list_of_evaluation_tables[-1] = list_of_evaluation_tables[-1].align( list_of_evaluation_tables[0], axis=1)[0] evaluation_table = pandas.concat(list_of_evaluation_tables, axis=0, ignore_index=True) print('Writing results to: "{0:s}"...'.format(output_file_name)) model_eval.write_evaluation( pickle_file_name=output_file_name, forecast_probabilities=forecast_probabilities, observed_labels=observed_labels, best_prob_threshold=best_prob_threshold, all_prob_thresholds=all_prob_thresholds, num_examples_by_forecast_bin=num_examples_by_forecast_bin, downsampling_dict=downsampling_dict, evaluation_table=evaluation_table)
def _find_examples_to_read(option_dict, num_examples_total): """Determines which examples to read. E = number of examples to read :param option_dict: See doc for any generator in this file. :param num_examples_total: Number of examples to generate. :return: storm_ids: length-E list of storm IDs (strings). :return: storm_times_unix_sec: length-E numpy array of storm times. """ error_checking.assert_is_integer(num_examples_total) error_checking.assert_is_greater(num_examples_total, 0) example_file_names = option_dict[trainval_io.EXAMPLE_FILES_KEY] radar_field_names = option_dict[trainval_io.RADAR_FIELDS_KEY] radar_heights_m_agl = option_dict[trainval_io.RADAR_HEIGHTS_KEY] first_storm_time_unix_sec = option_dict[trainval_io.FIRST_STORM_TIME_KEY] last_storm_time_unix_sec = option_dict[trainval_io.LAST_STORM_TIME_KEY] num_grid_rows = option_dict[trainval_io.NUM_ROWS_KEY] num_grid_columns = option_dict[trainval_io.NUM_COLUMNS_KEY] class_to_sampling_fraction_dict = option_dict[ trainval_io.SAMPLING_FRACTIONS_KEY] storm_ids = [] storm_times_unix_sec = numpy.array([], dtype=int) target_values = numpy.array([], dtype=int) target_name = None num_files = len(example_file_names) for i in range(num_files): print 'Reading target values from: "{0:s}"...'.format( example_file_names[i]) this_example_dict = input_examples.read_example_file( netcdf_file_name=example_file_names[i], include_soundings=False, radar_field_names_to_keep=[radar_field_names[0]], radar_heights_to_keep_m_agl=radar_heights_m_agl[[0]], first_time_to_keep_unix_sec=first_storm_time_unix_sec, last_time_to_keep_unix_sec=last_storm_time_unix_sec, num_rows_to_keep=num_grid_rows, num_columns_to_keep=num_grid_columns) target_name = this_example_dict[input_examples.TARGET_NAME_KEY] storm_ids += this_example_dict[input_examples.STORM_IDS_KEY] storm_times_unix_sec = numpy.concatenate(( storm_times_unix_sec, this_example_dict[input_examples.STORM_TIMES_KEY] )) target_values = numpy.concatenate(( target_values, this_example_dict[input_examples.TARGET_VALUES_KEY] )) indices_to_keep = numpy.where( target_values != target_val_utils.INVALID_STORM_INTEGER )[0] storm_ids = [storm_ids[k] for k in indices_to_keep] storm_times_unix_sec = storm_times_unix_sec[indices_to_keep] target_values = target_values[indices_to_keep] num_examples_found = len(storm_ids) if class_to_sampling_fraction_dict is None: indices_to_keep = numpy.linspace( 0, num_examples_found - 1, num=num_examples_found, dtype=int) if num_examples_found > num_examples_total: indices_to_keep = numpy.random.choice( indices_to_keep, size=num_examples_total, replace=False) else: indices_to_keep = dl_utils.sample_by_class( sampling_fraction_by_class_dict=class_to_sampling_fraction_dict, target_name=target_name, target_values=target_values, num_examples_total=num_examples_total) storm_ids = [storm_ids[k] for k in indices_to_keep] storm_times_unix_sec = storm_times_unix_sec[indices_to_keep] return storm_ids, storm_times_unix_sec