def __init__(self, Y_given_X, meta_feature, meta_weight, epsilon=None): """Construct a new probabilistic model. Args: Y_given_X: a function from ``X`` to an iterable object giving the subset of ``Y`` which has non-zero probability given the ``x``. When in doubt about whether some ``y`` has zero probability or not, it is always safe/correct to return a larger subset of ``Y`` (it'll just take more computation time is all). This is needed for computing the partition function and expectation. N.B., we do not actually need to know/enumerate of *all* of ``Y``, only the subsets for each ``x``. meta_feature (MetaFeature): A function ``X -> Y -> MetaFeatureValue``. N.B., for all ``x`` and ``y`` the length of ``wrapped_feature(x)(y)`` must be the same as the length of ``weights``. All features. meta_weight (MetaWeight): All weights. the weights for the features. The keys of the dictionary are the names of the feature that weight is for. We take this argument as a dict rather than as a list so that callers needn't worry about what order to provide the weights in. epsilon (float): The absolute-error threshold for considering a weight to be "equal to zero". N.B., this should be a positive number, as we will compare it against the absolute value of each weight. """ super(LogLinearModel, self).__init__(meta_feature, meta_weight, epsilon) self._Y = Y_given_X def _LogZ(x): score_given_x = self._scores(x) return logsumexp([score_given_x(y) for y in self._Y(x)]) self._zetas = MemoizedFunction(_LogZ)
def testClearMemos(self): """``MemoizedFunction._ClearMemos`` does actually clear the memos. That is, we call the underlying function once (to set the memo), then swap put the underlying function with a different one (to be sure we know whether the next ``__call__`` goes to the memos or to the function), and finally clear the memos and check. """ f = MemoizedFunction(_F) f(5) f._f = _G f.ClearMemos() self.assertEqual(_G(5), f(5))
def __init__(self, meta_feature, meta_weight, epsilon=None): """Construct a new model with the meta_feature and meta_weight. Args: meta_feature (MetaFeature): A function ``X -> Y -> MetaFeatureValue``. N.B., for all ``x`` and ``y`` the length of ``wrapped_feature(x)(y)`` must be the same as the length of ``weights``. All features. meta_weight (MetaWeight): All weights. the weights for the features. The keys of the dictionary are the names of the feature that weight is for. We take this argument as a dict rather than as a list so that callers needn't worry about what order to provide the weights in. epsilon (float): The absolute-error threshold for considering a weight to be "equal to zero". N.B., this should be a positive number, as we will compare it against the absolute value of each weight. """ self._epsilon = EPSILON if epsilon is None else epsilon self._meta_weight = meta_weight self._meta_weight.DropZeroWeights(self._epsilon) self._quadrance = None # TODO(crbug.com/674752): we need better names for ``self._features``. def _Features(x): """Wrap ``meta_feature`` to memoize things. This outer wrapping takes each ``x`` to a memoized instance of ``_FeaturesGivenX``. That is, for each ``x`` we return a ``MemoizedFunction`` from ``Y`` to ``dict(str to FeatureValue)``. """ # Memoize on ``Y``, to ensure we don't need to recompute # ``FeatureValue``s nor recheck the lengths. return MemoizedFunction(meta_feature(x)) # Memoize on ``X``, to ensure we share the memo tables on ``Y``. self._features = MemoizedFunction(_Features) # TODO(crbug.com/674752): we need better names for ``self._scores``. # N.B., this is just the inner product of ``self._meta_weight`` # against ``self._features(x)``. If we can compute this in some # more efficient way, we should. In particular, we will want to # make the weights sparse, in which case we need to use a sparse # variant of the dot product. self._scores = MemoizedFunction(lambda x: self._features(x).map( lambda fxy: self._meta_weight * fxy))
def _Features(x): """Wrap ``meta_feature`` to memoize things. This outer wrapping takes each ``x`` to a memoized instance of ``_FeaturesGivenX``. That is, for each ``x`` we return a ``MemoizedFunction`` from ``Y`` to ``dict(str to FeatureValue)``. """ # Memoize on ``Y``, to ensure we don't need to recompute # ``FeatureValue``s nor recheck the lengths. return MemoizedFunction(meta_feature(x))
def testNotToMemorizeIfXIsNotHashable(self): """``MemoizedFunction`` won't memorize when x is not hashable. We cannot always requir that the input of MemoizedFunction is hashable, that requirement would be too strict. So when the x is not hashable, we just bail out and run the internal function instead of getting result from memo.""" f = MemoizedFunction(_E) y = f({1: 3}) self.assertEqual(y, 4) self.assertEqual(f._memos, {})
def testMemoization(self): """``MemoizedFunction.__call__`` actually does memoize. That is, we call the underlying function once (to set the memo), then we discard the underlying function (to be sure the next ``__call__`` is handled from the memos, and finally call the function to check. """ f = MemoizedFunction(_F) f(5) del f._f self.assertEqual(_F(5), f(5))
def testMemoizedFunctionMap(self): """``MemoizedFunction.map`` composes functions as described.""" self.assertEqual(_G(_F(5)), MemoizedFunction(_F).map(_G)(5)) self.assertEqual(_F(_G(5)), MemoizedFunction(_G).map(_F)(5))
def testMemoizedFunctionCall(self): """``MemoizedFunction.__call__`` returns same value as its callable.""" self.assertEqual(_F(5), MemoizedFunction(_F)(5)) self.assertEqual(_G(5), MemoizedFunction(_G)(5))
class UnnormalizedLogLinearModel(object): """An unnormalized loglinear model. We use this model for combining a bunch of different features in order to decide who to blame. In particular, the nice thing about loglinear models is that (a) they allow separating the amount of weight we give to each feature from the feature itself, and (b) they allow us to keep adding features at will without needing to worry about the fiddly details needed for it to be a probability model. Throughout the documentation we use the following conventional terminology. The independent variable (aka: the given data which sets up the classification problem) is called ``X``, where particular values for that variable called ``x``. The dependent variable (aka: the answers/labels returned by classification) is called ``Y``, where particular values for that random variable called ``y``. The partition function is called ``Z``. And (somewhat non-conventionally) we will call the log of the partition function ``zeta``. This class is distinct from ``LogLinearModel`` in that we do not require a specification of ``Y``. This class is sufficient for determining which ``y`` is the most likely, though it can only return a "score" rather than returning a probability per se. """ def __init__(self, meta_feature, meta_weight, epsilon=None): """Construct a new model with the meta_feature and meta_weight. Args: meta_feature (MetaFeature): A function ``X -> Y -> MetaFeatureValue``. N.B., for all ``x`` and ``y`` the length of ``wrapped_feature(x)(y)`` must be the same as the length of ``weights``. All features. meta_weight (MetaWeight): All weights. the weights for the features. The keys of the dictionary are the names of the feature that weight is for. We take this argument as a dict rather than as a list so that callers needn't worry about what order to provide the weights in. epsilon (float): The absolute-error threshold for considering a weight to be "equal to zero". N.B., this should be a positive number, as we will compare it against the absolute value of each weight. """ self._epsilon = EPSILON if epsilon is None else epsilon self._meta_weight = meta_weight self._meta_weight.DropZeroWeights(self._epsilon) self._quadrance = None # TODO(crbug.com/674752): we need better names for ``self._features``. def _Features(x): """Wrap ``meta_feature`` to memoize things. This outer wrapping takes each ``x`` to a memoized instance of ``_FeaturesGivenX``. That is, for each ``x`` we return a ``MemoizedFunction`` from ``Y`` to ``dict(str to FeatureValue)``. """ # Memoize on ``Y``, to ensure we don't need to recompute # ``FeatureValue``s nor recheck the lengths. return MemoizedFunction(meta_feature(x)) # Memoize on ``X``, to ensure we share the memo tables on ``Y``. self._features = MemoizedFunction(_Features) # TODO(crbug.com/674752): we need better names for ``self._scores``. # N.B., this is just the inner product of ``self._meta_weight`` # against ``self._features(x)``. If we can compute this in some # more efficient way, we should. In particular, we will want to # make the weights sparse, in which case we need to use a sparse # variant of the dot product. self._scores = MemoizedFunction(lambda x: self._features(x).map( lambda fxy: self._meta_weight * fxy)) def ClearWeightBasedMemos(self): """Clear all the memos that depend on the weight covector.""" self._quadrance = None self._scores.ClearMemos() def ClearAllMemos(self): """Clear all memos, even those independent of the weight covector.""" self.ClearWeightBasedMemos() self._features.ClearMemos() @property def meta_weight(self): """The weight covector. At present we return the weights as an dict mapping feature name to its weight, but in the future that may be replaced by a more general type which specifies the semantics rather than the implementation details. """ return self._meta_weight @meta_weight.setter def meta_weight(self, new_meta_weight): self._meta_weight = new_meta_weight self._meta_weight.DropZeroWeights(self._epsilon) @property def l0(self): # pragma: no cover """The l0-norm of the weight covector. N.B., despite being popularly called the "l0-norm", this isn't actually a norm in the mathematical sense.""" return self._meta_weight.l0 @property def l1(self): # pragma: no cover """The l1 (aka: Manhattan) norm of the weight covector.""" return self._meta_weight.l1 @property def quadrance(self): """The square of the l2 norm of the weight covector. This value is often more helpful to have direct access to, as it avoids the need for non-rational functions (e.g., sqrt) and shows up as its own quantity in many places. Also, computing it directly avoids the error introduced by squaring the square-root of an IEEE-754 float. """ return self._meta_weight.quadrance @property def l2(self): """The l2 (aka Euclidean) norm of the weight covector. If you need the square of the l2-norm, do not use this property. Instead, use the ``quadrance`` property which is more accurate than squaring this one. """ return math.sqrt(self.quadrance) # TODO(crbug.com/673964): something better for detecting "close to log(0)". def LogZeroish(self, x): """Determine whether a float is close enough to log(0). If a ``FeatureValue`` has a (log-domain) score of -inf for a given ``Suspect``, then that suspect has zero probability of being the culprit. We want to filter these suspects out, to clean up the output of classification; so this method encapsulates the logic of that check. Args: x (float): the float to check Returns: ``True`` if ``x`` is close enough to log(0); else ``False``. """ return x < 0 and math.isinf(x) def Features(self, x): """Returns a function mapping ``y`` to its feature vector given ``x``. Args: x (X): the value of the independent variable. Returns: A ``MemoizedFunction`` of type ``Y -> dict(str to float)``. """ return self._features(x) def Score(self, x): """Returns a function mapping ``y`` to its "score" given ``x``. Semantically, the "score" of ``y`` given ``x`` is the unnormalized log-domain conditional probability of ``y`` given ``x``. Operationally, we compute this by taking the inner product of the weight covector with the ``Features(x)(y)`` vector. The conditional probability itself can be computed by exponentiating the "score" and normalizing with the partition function. However, for determining the "best" ``y`` we don't need to bother computing the probability, because ``Score(x)`` is monotonic with respect to ``Probability(x)``. Args: x (X): the value of the independent variable. Returns: A ``MemoizedFunction`` of type ``Y -> float``. """ return self._scores(x) def FilterReasonWithWeight(self, reason): """Filters reasons with zero weights. Args: reason (dict): Dict mapping feature name to reason list. Returns: A list of reason strings. """ flat_weight = self._meta_weight.leaves filtered_reasons = {} for non_zero_feature in flat_weight: if non_zero_feature in reason: filtered_reasons[non_zero_feature] = reason[non_zero_feature] sorted_reasons = sorted( filtered_reasons.items(), key=lambda item: -_FEATURE_TO_REASON_PRIORITY.get( item[0], -float('inf'))) reasons = [] for _, reason in sorted_reasons: reasons.extend(reason) return reasons
class LogLinearModel(UnnormalizedLogLinearModel): """A loglinear probability model. This class is distinct from ``UnnormalizedLogLinearModel`` in that we can provide probabilities (not just scores). However, to do so we require a specification of the subsets of ``Y`` for each ``x``. """ def __init__(self, Y_given_X, meta_feature, meta_weight, epsilon=None): """Construct a new probabilistic model. Args: Y_given_X: a function from ``X`` to an iterable object giving the subset of ``Y`` which has non-zero probability given the ``x``. When in doubt about whether some ``y`` has zero probability or not, it is always safe/correct to return a larger subset of ``Y`` (it'll just take more computation time is all). This is needed for computing the partition function and expectation. N.B., we do not actually need to know/enumerate of *all* of ``Y``, only the subsets for each ``x``. meta_feature (MetaFeature): A function ``X -> Y -> MetaFeatureValue``. N.B., for all ``x`` and ``y`` the length of ``wrapped_feature(x)(y)`` must be the same as the length of ``weights``. All features. meta_weight (MetaWeight): All weights. the weights for the features. The keys of the dictionary are the names of the feature that weight is for. We take this argument as a dict rather than as a list so that callers needn't worry about what order to provide the weights in. epsilon (float): The absolute-error threshold for considering a weight to be "equal to zero". N.B., this should be a positive number, as we will compare it against the absolute value of each weight. """ super(LogLinearModel, self).__init__(meta_feature, meta_weight, epsilon) self._Y = Y_given_X def _LogZ(x): score_given_x = self._scores(x) return logsumexp([score_given_x(y) for y in self._Y(x)]) self._zetas = MemoizedFunction(_LogZ) def ClearWeightBasedMemos(self): """Clear all the memos that depend on the weight covector.""" super(LogLinearModel, self).ClearWeightBasedMemos() self._zetas.ClearMemos() def Z(self, x): """The normal-domain conditional partition-function given ``x``. If you need the log of the partition function, do not use this method. Instead, use the ``LogZ`` method which computes it directly and thus is more accurate than taking the log of this one. Args: x (X): the value of the independent variable. Returns: The normalizing constant of ``Probability(x)``. """ return math.exp(self.LogZ(x)) def LogZ(self, x): """The log-domain conditional partition-function given ``x``. Args: x (X): the value of the independent variable. Returns: The normalizing constant of ``LogProbability(x)``. """ return self._zetas(x) def Probability(self, x): """The normal-domain distribution over ``y`` given ``x``. That is, ``self.Probability(x)(y)`` returns ``p(y | x; self._meta_weight)`` which is the model's estimation of ``Pr(y|x)``. If you need the log-probability, don't use this method. Instead, use the ``LogProbability`` method which computes it directly and thus is more accurate than taking the log of this one. """ logprob_given_x = self.LogProbability(x) return lambda y: math.exp(logprob_given_x(y)) def LogProbability(self, x): """The log-domain distribution over ``y`` given ``x``. That is, ``self.LogProbability(x)(y)`` returns the log of ``self.Probability(x)(y)``. However, this method computes the log-probabilities directly and so is more accurate and efficient than taking the log of ``Probability``. """ zeta_x = self.LogZ(x) score_given_x = self.Score(x) return lambda y: score_given_x(y) - zeta_x def Expectation(self, x, f): """Compute the expectation of a function with respect to ``Probability(x)``. Args: x (X): the value of the independent variable. f: a function of type ``Y -> np.ndarray(float)``. Returns: An ``np.ndarray`` of ``float`` called the "expected value". N.B., the particular array returned may not actually be one that the function returns; rather, it's a sort of average of all the results returned. For more information you can take a look at Wikipedia <https://en.wikipedia.org/wiki/Expected_value>. """ prob_given_x = self.Probability(x) # N.B., the ``*`` below is vector scaling! If we want to make this # method polymorphic in the return type of ``f`` then we'll need an # API that provides both scaling and ``vsum``. return vsum([prob_given_x(y) * f(y) for y in self._Y(x)])