def testCalculateConfidenceInterval(self): sampling_data_list = [ np.array([ [0, 0, 2, 7, 0.77777779, 1], [1, 0, 2, 6, 0.75, 0.85714287], [4, 0, 2, 3, 0.60000002, 0.42857143], [4, 2, 0, 3, 1, 0.42857143], [7, 2, 0, 0, float('nan'), 0], ]), np.array([ [7, 2, 0, 0, float('nan'), 0], [0, 0, 2, 7, 0.77777779, 1], [1, 0, 2, 6, 0.75, 0.85714287], [4, 0, 2, 3, 0.60000002, 0.42857143], [4, 2, 0, 3, 1, 0.42857143], ]), ] unsampled_data = np.array([ [4, 2, 0, 3, 1, 0.42857143], [7, 2, 0, 0, float('nan'), 0], [0, 0, 2, 7, 0.77777779, 1], [1, 0, 2, 6, 0.75, 0.85714287], [4, 0, 2, 3, 0.60000002, 0.42857143], ]) result = aggregate._calculate_confidence_interval( sampling_data_list, unsampled_data) print(result) self.assertIsInstance(result, np.ndarray) self.assertEqual(result.shape, (5, 6)) self.assertAlmostEqual(result[0][0].value, 3.5, delta=0.1) self.assertAlmostEqual(result[0][0].lower_bound, -40.97, delta=0.1) self.assertAlmostEqual(result[0][0].upper_bound, 47.97, delta=0.1) self.assertEqual(result[0][0].unsampled_value, 4.0) self.assertAlmostEqual(result[0][4].value, 0.77, delta=0.1) self.assertTrue(np.isnan(result[0][4].lower_bound)) self.assertTrue(np.isnan(result[0][4].upper_bound)) self.assertEqual(result[0][4].unsampled_value, 1.0) sampling_data_list = [ np.array([1, 2]), np.array([1, 2]), np.array([1, float('nan')]) ] unsampled_data = np.array([1, 2]) result = aggregate._calculate_confidence_interval( sampling_data_list, unsampled_data) self.assertIsInstance(result, np.ndarray) self.assertEqual(result.tolist(), [ types.ValueWithConfidenceInterval( value=1.0, lower_bound=1.0, upper_bound=1.0, unsampled_value=1), types.ValueWithConfidenceInterval( value=2.0, lower_bound=2.0, upper_bound=2.0, unsampled_value=2) ])
def testUncertaintyValuedMetrics(self): slice_key = _make_slice_key() slice_metrics = { 'one_dim': types.ValueWithConfidenceInterval(2.0, 1.0, 3.0), 'nans': types.ValueWithConfidenceInterval(float('nan'), float('nan'), float('nan')), } expected_metrics_for_slice = text_format.Parse( """ slice_key {} metrics { key: "one_dim" value { bounded_value { value { value: 2.0 } lower_bound { value: 1.0 } upper_bound { value: 3.0 } methodology: POISSON_BOOTSTRAP } } } metrics { key: "nans" value { bounded_value { value { value: nan } lower_bound { value: nan } upper_bound { value: nan } methodology: POISSON_BOOTSTRAP } } } """, metrics_for_slice_pb2.MetricsForSlice()) got = metrics_and_plots_evaluator._serialize_metrics( (slice_key, slice_metrics), []) self.assertProtoEquals( expected_metrics_for_slice, metrics_for_slice_pb2.MetricsForSlice.FromString(got))
def _calculate_confidence_interval(sampling_data_list: List[Union[int, float, np.ndarray]], unsampled_data: Union[int, float, np.ndarray]): """Calculate the confidence interval of the data. Args: sampling_data_list: A list of number or np.ndarray. unsampled_data: Individual number or np.ndarray. The format of the unsampled_data should match the format of the element inside sampling_data_list. Returns: Confidence Interval value stored inside types.ValueWithConfidenceInterval. """ if isinstance(sampling_data_list[0], (np.ndarray, list)): merged_data = sampling_data_list[0][:] if isinstance(sampling_data_list[0], np.ndarray): merged_data = merged_data.astype(object) for index in range(len(merged_data)): merged_data[index] = _calculate_confidence_interval( [data[index] for data in sampling_data_list], unsampled_data[index]) return merged_data else: # Data has to be numeric. That means throw out nan values. sampling_data_list = [ data for data in sampling_data_list if not np.isnan(data) ] n_samples = len(sampling_data_list) if n_samples: confidence = 0.95 sample_mean = mean(sampling_data_list) std_err = sem(sampling_data_list) t_stat = t.ppf((1 + confidence) / 2, n_samples - 1) upper_bound = sample_mean + t_stat * std_err lower_bound = sample_mean - t_stat * std_err # Set [mean, lower_bound, upper_bound] for each metric component. return types.ValueWithConfidenceInterval(sample_mean, lower_bound, upper_bound, unsampled_data) else: return types.ValueWithConfidenceInterval(float('nan'), float('nan'), float('nan'), float('nan'))
def testSerializeMetricsRanges(self): slice_key = _make_slice_key('age', 5, 'language', 'english', 'price', 0.3) slice_metrics = { 'accuracy': types.ValueWithConfidenceInterval(0.8, 0.7, 0.9), _full_key(metric_keys.AUPRC): 0.1, _full_key(metric_keys.lower_bound(metric_keys.AUPRC)): 0.05, _full_key(metric_keys.upper_bound(metric_keys.AUPRC)): 0.17, _full_key(metric_keys.AUC): 0.2, _full_key(metric_keys.lower_bound(metric_keys.AUC)): 0.1, _full_key(metric_keys.upper_bound(metric_keys.AUC)): 0.3 } expected_metrics_for_slice = text_format.Parse( string.Template(""" slice_key { single_slice_keys { column: 'age' int64_value: 5 } single_slice_keys { column: 'language' bytes_value: 'english' } single_slice_keys { column: 'price' float_value: 0.3 } } metrics { key: "accuracy" value { bounded_value { value { value: 0.8 } lower_bound { value: 0.7 } upper_bound { value: 0.9 } methodology: POISSON_BOOTSTRAP } } } metrics { key: "$auc" value { bounded_value { lower_bound { value: 0.1 } upper_bound { value: 0.3 } value { value: 0.2 } methodology: RIEMANN_SUM } } } metrics { key: "$auprc" value { bounded_value { lower_bound { value: 0.05 } upper_bound { value: 0.17 } value { value: 0.1 } methodology: RIEMANN_SUM } } }""").substitute(auc=_full_key(metric_keys.AUC), auprc=_full_key(metric_keys.AUPRC)), metrics_for_slice_pb2.MetricsForSlice()) got = metrics_and_plots_evaluator._serialize_metrics( (slice_key, slice_metrics), [post_export_metrics.auc(), post_export_metrics.auc(curve='PR')]) self.assertProtoEquals( expected_metrics_for_slice, metrics_for_slice_pb2.MetricsForSlice.FromString(got))
def process(self, element, unsampled_results): slice_key, metrics = element # metrics should be a list of dicts, but the dataflow runner has a quirk # that requires specific casting. metrics = list(metrics) side_input_results = {} for result in unsampled_results: unsampled_slice_key, unsampled_metrics = result side_input_results[unsampled_slice_key] = unsampled_metrics if len(metrics) == 1: yield slice_key, metrics[0] return original_structure = copy.copy(metrics[0]) uber_metrics = {} unsampled_metrics = {} for m_dict in metrics: # For each metric in each slice, aggregate values over all of the computed # samples. for key in m_dict: _collect_metrics(m_dict[key], key, uber_metrics) unsampled_slice_key = slice_key _collect_metrics(side_input_results[unsampled_slice_key][key], key, unsampled_metrics) for key in uber_metrics: # Compute confidence interval given the data points per metric. confidence = 0.95 data = uber_metrics[key] # Data has to be numeric. That means throw out nan values. n_samples = len(data) if n_samples: sample_mean = mean(data) std_err = sem(data) t_stat = t.ppf((1 + confidence) / 2, n_samples - 1) upper_bound = sample_mean + t_stat * std_err lower_bound = sample_mean - t_stat * std_err # Set [mean, lower_bound, upper_bound] for each metric component. uber_metrics[key] = types.ValueWithConfidenceInterval( sample_mean, lower_bound, upper_bound, unsampled_metrics[key][0]) else: uber_metrics[key] = types.ValueWithConfidenceInterval( float('nan'), float('nan'), float('nan'), float('nan')) # Convert metrics back into expected format with bounded values. for sub_key in uber_metrics: # Break sub-key into components. key_components = sub_key.split(',') original_key = key_components[0] metric_structure = original_structure[original_key] if isinstance(metric_structure, np.ndarray): metric_structure = np.array(metric_structure, dtype=object) _populate_bounded_metrics(key_components[1:], metric_structure, uber_metrics[sub_key]) else: metric_structure = uber_metrics[sub_key] original_structure[original_key] = metric_structure yield slice_key, original_structure