def entropy_from_column_summary(summary: ColumnSummary, histogram: datasketches.kll_floats_sketch): """ Calculate the estimated entropy for a ColumnProfile, using the ColumnSummary Can be used for both continuous and discrete types of data. Parameters ---------- summary : ColumnSummary Protobuf summary message histogram: datasketches.kll_floats_sketch Data sketch for quantiles Returns ------- entropy : float Estimated entropy value, np.nan if the inferred data type of the column is not categorical or numeric """ frequent_items = summary.frequent_items unique_count = summary.unique_count.estimate inferred_type = summary.schema.inferred_type.type total_count = summary.counters.count if inferred_type == InferredType.Type.FRACTIONAL: if histogram.get_min_value() == histogram.get_max_value() or histogram.get_n() <= 1: return 0 bins = np.linspace(histogram.get_min_value(), histogram.get_max_value(), 100) pmf = histogram.get_pmf(bins) pmf = list(filter(lambda x: x > 0, pmf)) entropy = -np.sum(pmf * np.log(pmf)) return entropy elif inferred_type in (InferredType.Type.INTEGRAL, InferredType.Type.STRING, InferredType.Type.BOOLEAN): if total_count == 0: return 0 entropy = 0 for item in frequent_items.items: i_frequency = item.estimate / total_count entropy += i_frequency * np.log(i_frequency) frequent_items_count = len(frequent_items.items) n_singles = unique_count - frequent_items_count if math.isclose(n_singles, 0.0, abs_tol=10e-3): return -entropy n_singles_frequency = n_singles / total_count entropy += n_singles_frequency * np.log(n_singles_frequency) return -entropy return np.nan
def _compute_kl_divergence_continuous_distributions(target_distribution: kll_floats_sketch, reference_distribution: kll_floats_sketch): """ Calculates the estimated KL divergence for two continuous distributions. Uses the `datasketches.kll_floats_sketch` sketch to calculate the KL divergence based on the PMFs. Only applicable to continuous distributions. Parameters ---------- target_distribution : datasketches.kll_floats_sketch The quantiles summary of the target feature's distribution. reference_distribution : datasketches.kll_floats_sketch The quantiles summary of the reference feature's distribution. Returns ------- kl_divergence : float The estimated KL divergence between two continuous features. """ almost_zero_probability_of_event = 10e-5 bins_target = np.linspace(target_distribution.get_min_value(), target_distribution.get_max_value(), 100) pmf_target = np.array(target_distribution.get_pmf(bins_target)) pmf_reference = np.array(reference_distribution.get_pmf(bins_target)) pmf_reference[pmf_reference == 0] = almost_zero_probability_of_event kl_divergence = np.sum(np.where(pmf_target != 0, pmf_target * np.log(pmf_target / pmf_reference), 0)) return type("Object", (), {"kl_divergence": kl_divergence})
def histogram_from_sketch(sketch: kll_floats_sketch, max_buckets: int = None, avg_per_bucket: int = None): """ Generate a summary of a kll_floats_sketch, including a histogram Parameters ---------- sketch : kll_floats_sketch Data sketch max_buckets : int Override the default maximum number of buckets avg_per_bucket : int Override the default target number of items per bucket. Returns ------- histogram : HistogramSummary Protobuf histogram message """ n = sketch.get_n() start = sketch.get_min_value() max_val = sketch.get_max_value() end = max_val if max_buckets is None: max_buckets = MAX_HIST_BUCKETS if avg_per_bucket is None: avg_per_bucket = HIST_AVG_NUMBER_PER_BUCKET if (n < 2) or (start == end): dx = abs(start) * 1e-7 end = start + dx bins = [start, end] counts = [n] else: # Include the max value in the right-most bin end += abs(end) * (1e-7) # Include the right edge in the bin edges n_buckets = min(math.ceil(n / HIST_AVG_NUMBER_PER_BUCKET), MAX_HIST_BUCKETS) width = (end - start) / n_buckets # Calculate histograms from the Probability Mass Function bins = [start + i * width for i in range(n_buckets + 1)] pmf = sketch.get_pmf(bins) counts = [round(p * n) for p in pmf] counts = counts[1:-1] return HistogramSummary( start=start, end=end, width=0, counts=counts, max=max_val, min=start, bins=bins, n=n, )