def testCalculatePartialExpectedMutualInformation(self):
# The two values co-occur in all observations, EMI is 0.
self.assertNear(
info_theory.calculate_partial_expected_mutual_information(10, 10, 10),
0, EPSILON)
# The two values co-occur no observations, EMI is 0
self.assertNear(
info_theory.calculate_partial_expected_mutual_information(10, 0, 0), 0,
EPSILON)
# The two values each appear 50% of the time.
self.assertNear(
info_theory.calculate_partial_expected_mutual_information(10, 5, 5),
.215411, EPSILON)
# The two values have differing frequencies.
self.assertNear(
info_theory.calculate_partial_expected_mutual_information(10, 2, 4),
0.524209, EPSILON)
def _calculate_mutual_information_for_feature_value(feature_and_accumulator,
global_accumulator,
use_adjusted_mutual_info,
min_diff_from_avg):
"""Calculates the (possibly adjusted) mutual information of a feature value.
Used as a measure of relatedness between a single feature value and a label.
Mutual information is calculated as:
H(x, y) = (sum(weights) *
[(P(y|x)*log2(P(y|x)/P(y))) + (P(~y|x)*log2(P(~y|x)/P(~y)))])
where x is feature and y is label. We use sum(weights) instead of P(x), as
this makes the mutual information more interpretable.
If we don't divide by sum(weights), it can be thought of as an adjusted
weighted count.
If use_adjusted_mutual_info is True, we use Adjusted Mutual Information (AMI)
which accounts for relatedness due to chance. AMI is generally calculated as:
AMI(x, y) = MI(x, y) - EMI(x, y) / (max(H(x), H(y)) - EMI(x, y))
where x is the feature and y is label. Here, we leave off the normalization
and only subtract expected mutual information (EMI) from mutual information.
The calculation is based on the following paper:
Vinh, N. X.; Epps, J.; Bailey, J. (2009). "Information theoretic measures for
clusterings comparison". Proceedings of the 26th Annual International Confere
nce on Machine Learning - ICML '09. p. 1.
doi:10.1145/1553374.1553511. ISBN 9781605585161.
Short summary can be found in the Wikipedia link:
https://en.wikipedia.org/wiki/Adjusted_mutual_information
Args:
feature_and_accumulator: A tuple of the form:
(feature, WeightedMeanAndVarCombiner.accumulator_class) where: `feature`
is the single token in the vocabulary for which (possibly adjusted)
mutual information with the label is being computed. `mean` is the
weighted mean positive for each label value given x. `count` is the
count of weights for a feature. `weight` is the mean of the weights for
a feature.
global_accumulator: A WeightedMeanAndVarCombiner.accumulator_class where:
`mean` is the weighted mean positive for each label value for all
features. `count` is the count for all features. `weight` is the mean of
the weights for all features.
use_adjusted_mutual_info: If set to True, use adjusted mutual information.
min_diff_from_avg: A regularization parameter that pushes low MI/AMI towards
zero. The Mutual information of a feature x label pair will be adjusted to
zero whenever the absolute difference the weight and the expected
(average) weight is lower than min_diff_from_average.
Returns:
A tuple of:
The feature value
The mutual information with the label. If use_adjusted_mutual_info, this
is the mutual information - the expected mutual information, otherwise
it is the raw mutual information.
The expected mutual information (EMI) if use_adjusted_mutual_info is
True, otherwise NaN.
"""
# Compute the frequency of each label value.
global_label_counts = (
global_accumulator.mean * global_accumulator.weight *
global_accumulator.count)
feature_value, current_accumulator = feature_and_accumulator
n = sum(global_label_counts)
if n == 0:
return (feature_value, float('NaN'), float('NaN'))
mutual_information = 0
expected_mutual_information = 0 if use_adjusted_mutual_info else None
x_i = (current_accumulator.count * current_accumulator.weight)
# If x_i == n, the feature is a constant and thus has no information.
if round(x_i) == round(n):
return feature_value, 0, 0
if x_i > n:
raise ValueError(
'Frequency of token {} higher than number of records {} > {}'.format(
feature_value, x_i, n) +
' This likely means you have provided tft.vocabulary with input that'
' has repeated tokens per row, rather than a set representation.')
for label_ix in range(len(global_label_counts)):
y_i = global_label_counts[label_ix]
if y_i == 0:
continue
local_mean = 0
if label_ix < len(current_accumulator.mean):
local_mean = current_accumulator.mean[label_ix]
n_i = (
_clip_probability(local_mean) * current_accumulator.weight *
current_accumulator.count)
diff_from_avg = (x_i * y_i / n) - n_i
if abs(diff_from_avg) < min_diff_from_avg:
continue
mutual_information += (
info_theory.calculate_partial_mutual_information(n_i, x_i, y_i, n))
if use_adjusted_mutual_info:
expected_mutual_information += (
info_theory.calculate_partial_expected_mutual_information(
n, x_i, y_i))
if use_adjusted_mutual_info:
# TODO(b/127366670): Consider implementing the normalization step as per
# AMI(x, y) = MI(x, y) - EMI(x, y) / (max(H(x), H(y)) - EMI(x, y))
return (feature_value, mutual_information - expected_mutual_information,
expected_mutual_information)
else:
return (feature_value, mutual_information, float('NaN'))
def _calculate_mutual_information_for_binary_feature(feature_and_accumulator,
global_accumulator,
use_adjusted_mutual_info,
min_diff_from_avg):
"""Calculates the (possibly adjusted) mutual information of a binary feature.
Used as a measure of relatedness between a binary feature and binary label.
Mutual information is calculated as:
H(x, y) = (sum(weights) *
[(P(y|x)*log2(P(y|x)/P(y))) + (P(~y|x)*log2(P(~y|x)/P(~y)))])
where x is feature and y is label. We use sum(weights) instead of P(x), as
this makes the mutual information more interpretable.
If we don't divide by sum(weights), it can be thought of as an adjusted
weighted count.
If use_adjusted_mutual_info is True, we use Adjusted Mutual Information (AMI)
which accounts for relatedness due to chance. AMI is generally calculated as:
AMI(x, y) = MI(x, y) - EMI(x, y) / (max(H(x), H(y)) - EMI(x, y))
where x is the feature and y is label. Here, we leave off the normalization
and only subtract expected mutual information (EMI) from mutual information.
The calculation is based on the following paper:
Vinh, N. X.; Epps, J.; Bailey, J. (2009). "Information theoretic measures for
clusterings comparison". Proceedings of the 26th Annual International Confere
nce on Machine Learning - ICML '09. p. 1.
doi:10.1145/1553374.1553511. ISBN 9781605585161.
Short summary can be found in the Wikipedia link:
https://en.wikipedia.org/wiki/Adjusted_mutual_information
Args:
feature_and_accumulator: A tuple of the form:
(feature, _CountAndWeightsMeansAccumulator) where: `feature` is a single
string, which is the word in the vocabulary whose mutual information
with the label is being computed. `weighted_mean` is the weighted mean
positive given x. `count` is the count of weights for a feature.
`weights_mean` is the mean of the weights for a feature.
global_accumulator: A _CountAndWeightsMeansAccumulator where:
`weighted_mean` is the weighted mean of positive labels for all features.
`count` is the count for all features. `mean` is the mean of the weights
for all features.
use_adjusted_mutual_info: If set to True, use adjusted mutual information.
min_diff_from_avg: Mutual information of a feature will be adjusted to zero
whenever the absolute difference between count of the feature with any
label and its expected count is lower than min_diff_from_average.
Returns:
The feature and its mutual information.
"""
feature, current_accumulator = feature_and_accumulator
x = (current_accumulator.count * current_accumulator.weights_mean)
n = (global_accumulator.count * global_accumulator.weights_mean)
if n == 0:
return (feature, float('NaN'))
n_1, n_0 = [
weighted_mean * current_accumulator.weights_mean *
current_accumulator.count for weighted_mean in _clip_probability(
current_accumulator.weighted_mean)
]
y_1, y_0 = [
weighted_mean * global_accumulator.weights_mean *
global_accumulator.count for weighted_mean in _clip_probability(
global_accumulator.weighted_mean)
]
diff_from_avg = x * y_1 / n - n_1
if abs(diff_from_avg) < min_diff_from_avg:
return (feature, 0)
mutual_information = (
n_1 * (np.log2(n_1) + np.log2(n) - np.log2(y_1) - np.log2(x)) + n_0 *
(np.log2(n_0) + np.log2(n) - np.log2(y_0) - np.log2(x)))
if use_adjusted_mutual_info:
# Note: Expected mutual information is calculated by summing over all values
# of x and all values of y, but here we don't count the contribution of the
# case where the feature is not present, and this is consistent with
# how mutual information is computed.
expected_mutual_information = (
info_theory.calculate_partial_expected_mutual_information(
n, x, y_1) +
info_theory.calculate_partial_expected_mutual_information(
n, x, y_0))
# TODO(b/127366670): Consider implementing the normalization step as per
# AMI(x, y) = MI(x, y) - EMI(x, y) / (max(H(x), H(y)) - EMI(x, y))
return (feature, mutual_information - expected_mutual_information)
else:
return (feature, mutual_information)
def test_calculate_partial_expected_mutual_information(
self, n, x_i, y_j, expected):
self.assertNear(
info_theory.calculate_partial_expected_mutual_information(
n, x_i, y_j), expected, EPSILON)