def __init__( self, # pylint: disable=useless-super-delegation name = 'NumericStatsGenerator', schema = None, num_histogram_buckets = 10, num_quantiles_histogram_buckets = 10, epsilon = 0.01): """Initializes a numeric statistics generator. Args: name: An optional unique name associated with the statistics generator. schema: An optional schema for the dataset. num_histogram_buckets: An optional number of buckets in a standard NumericStatistics.histogram with equal-width buckets. num_quantiles_histogram_buckets: An optional number of buckets in a quantiles NumericStatistics.histogram. epsilon: An optional error tolerance for the computation of quantiles, typically a small fraction close to zero (e.g. 0.01). Higher values of epsilon increase the quantile approximation, and hence result in more unequal buckets, but could improve performance, and resource consumption. """ super(NumericStatsGenerator, self).__init__(name) self._categorical_features = set( stats_util.get_categorical_numeric_features(schema) if schema else []) # Initialize quantiles combiner for equi-width histogram. self._std_hist_combiner = quantiles_util.QuantilesCombiner( _NUM_QUANTILES_FACTOR_FOR_STD_HISTOGRAM * num_histogram_buckets, epsilon) # Initialize quantiles combiner for quantiles histogram. self._quantiles_hist_combiner = quantiles_util.QuantilesCombiner( num_quantiles_histogram_buckets, epsilon) self._num_histogram_buckets = num_histogram_buckets
def __init__( self, # pylint: disable=useless-super-delegation name: Text = 'BasicStatsGenerator', schema: Optional[schema_pb2.Schema] = None, weight_feature: Optional[types.FeatureName] = None, num_values_histogram_buckets: Optional[int] = 10, num_histogram_buckets: Optional[int] = 10, num_quantiles_histogram_buckets: Optional[int] = 10, epsilon: Optional[float] = 0.01) -> None: """Initializes basic statistics generator. Args: name: An optional unique name associated with the statistics generator. schema: An optional schema for the dataset. weight_feature: An optional feature name whose numeric value represents the weight of an example. num_values_histogram_buckets: An optional number of buckets in a quantiles histogram for the number of values per Feature, which is stored in CommonStatistics.num_values_histogram. num_histogram_buckets: An optional number of buckets in a standard NumericStatistics.histogram with equal-width buckets. num_quantiles_histogram_buckets: An optional number of buckets in a quantiles NumericStatistics.histogram. epsilon: An optional error tolerance for the computation of quantiles, typically a small fraction close to zero (e.g. 0.01). Higher values of epsilon increase the quantile approximation, and hence result in more unequal buckets, but could improve performance, and resource consumption. """ super(BasicStatsGenerator, self).__init__(name, schema) self._bytes_features = set( schema_util.get_bytes_features(schema) if schema else []) self._categorical_features = set( schema_util.get_categorical_numeric_features(schema ) if schema else []) self._weight_feature = weight_feature self._num_values_histogram_buckets = num_values_histogram_buckets # Initialize quantiles combiner for histogram over number of values. self._num_values_quantiles_combiner = quantiles_util.QuantilesCombiner( self._num_values_histogram_buckets, epsilon) self._num_histogram_buckets = num_histogram_buckets self._num_quantiles_histogram_buckets = num_quantiles_histogram_buckets num_buckets = max( self._num_quantiles_histogram_buckets, _NUM_QUANTILES_FACTOR_FOR_STD_HISTOGRAM * self._num_histogram_buckets) assert num_buckets % self._num_quantiles_histogram_buckets == 0 # Initialize quantiles combiner for histogram over feature values. self._values_quantiles_combiner = quantiles_util.QuantilesCombiner( num_buckets, epsilon, has_weights=True)
def __init__( self, # pylint: disable=useless-super-delegation name='CommonStatsGenerator', schema=None, weight_feature=None, num_values_histogram_buckets=10, epsilon=0.01): """Initializes a common statistics generator. Args: name: An optional unique name associated with the statistics generator. schema: An optional schema for the dataset. weight_feature: An optional feature name whose numeric value represents the weight of an example. num_values_histogram_buckets: An optional number of buckets in a quantiles histogram for the number of values per Feature, which is stored in CommonStatistics.num_values_histogram. epsilon: An optional error tolerance for the computation of quantiles, typically a small fraction close to zero (e.g. 0.01). Higher values of epsilon increase the quantile approximation, and hence result in more unequal buckets, but could improve performance, and resource consumption. """ super(CommonStatsGenerator, self).__init__(name, schema) self._categorical_features = set( schema_util.get_categorical_numeric_features(schema ) if schema else []) self._weight_feature = weight_feature self._num_values_histogram_buckets = num_values_histogram_buckets # Initialize quantiles combiner. self._quantiles_combiner = quantiles_util.QuantilesCombiner( self._num_values_histogram_buckets, epsilon)
def test_quantiles_combiner(self): batches = [[np.linspace(1, 100, 100)], [np.linspace(101, 200, 100)], [np.linspace(201, 300, 100)]] expected_result = np.array([61.0, 121.0, 181.0, 241.0], dtype=np.float32) q_combiner = quantiles_util.QuantilesCombiner(5, 0.00001) _run_quantiles_combiner_test(self, q_combiner, batches, expected_result)
def test_quantiles_combiner_with_weights(self): batches = [[np.linspace(1, 100, 100), [1] * 100], [np.linspace(101, 200, 100), [2] * 100], [np.linspace(201, 300, 100), [3] * 100]] expected_result = np.array( [1.0, 111.0, 171.0, 221.0, 261.0, 300.0], dtype=np.float32) q_combiner = quantiles_util.QuantilesCombiner(5, 0.00001, has_weights=True) _run_quantiles_combiner_test(self, q_combiner, batches, expected_result)