def quantiles_from_sketch(sketch: kll_floats_sketch, quantiles=None):
"""
Calculate quantiles from a data sketch
Parameters
----------
sketch : kll_floats_sketch
Data sketch
quantiles : list-like
Override the default quantiles. Should be a list of values from
0 to 1 inclusive.
"""
if quantiles is None:
quantiles = QUANTILES
qvals = sketch.get_quantiles(quantiles)
return QuantileSummary(
quantiles=quantiles,
quantile_values=qvals,
)
def single_quantile_from_sketch(sketch: kll_floats_sketch, quantile: float):
"""
Calculate the specified quantile from a data sketch
Parameters
----------
sketch : kll_floats_sketch
Data sketch
quantile : float
Override the default quantiles to a single quantile. Should be a value from
0 to 1 inclusive.
Returns
----------
Anonymous object with one filed equal to the quantile value
"""
if quantile is None:
raise ValueError("The quantile value is required and should be of type float")
qval = sketch.get_quantiles([quantile])
return type("Object", (), {"quantile": qval[0]})
def ks_test_compute_p_value(target_distribution: kll_floats_sketch, reference_distribution: kll_floats_sketch):
"""
Compute the Kolmogorov-Smirnov test p-value of two continuous distributions.
Uses the quantile values and the corresponding CDFs to calculate the approximate KS statistic.
Only applicable to continuous distributions.
The null hypothesis expects the samples to come from the same distribution.
Parameters
----------
target_distribution : datasketches.kll_floats_sketch
A kll_floats_sketch (quantiles sketch) from the target distribution's values
reference_distribution : datasketches.kll_floats_sketch
A kll_floats_sketch (quantiles sketch) from the reference (expected) distribution's values
Can be generated from a theoretical distribution, or another sample for the same feature.
Returns
-------
p_value : float
The estimated p-value from the parametrized KS test, applied on the target and reference distributions'
kll_floats_sketch summaries
"""
D_max = 0
target_quantile_values = target_distribution.get_quantiles(QUANTILES)
ref_quantile_values = reference_distribution.get_quantiles(QUANTILES)
num_quantiles = len(QUANTILES)
i, j = 0, 0
while i < num_quantiles and j < num_quantiles:
if target_quantile_values[i] < ref_quantile_values[j]:
current_quantile = target_quantile_values[i]
i += 1
else:
current_quantile = ref_quantile_values[j]
j += 1
cdf_target = target_distribution.get_cdf([current_quantile])[0]
cdf_ref = reference_distribution.get_cdf([current_quantile])[0]
D = abs(cdf_target - cdf_ref)
if D > D_max:
D_max = D
while i < num_quantiles:
cdf_target = target_distribution.get_cdf([target_quantile_values[i]])[0]
cdf_ref = reference_distribution.get_cdf([target_quantile_values[i]])[0]
D = abs(cdf_target - cdf_ref)
if D > D_max:
D_max = D
i += 1
while j < num_quantiles:
cdf_target = target_distribution.get_cdf([ref_quantile_values[j]])[0]
cdf_ref = reference_distribution.get_cdf([ref_quantile_values[j]])[0]
D = abs(cdf_target - cdf_ref)
if D > D_max:
D_max = D
j += 1
m, n = sorted([target_distribution.get_n(), reference_distribution.get_n()], reverse=True)
en = m * n / (m + n)
p_value = stats.distributions.kstwo.sf(D_max, np.round(en))
return type("Object", (), {"ks_test": p_value})