def value_and_gradient(self, weights, verbose=False):
if weights == self.last_vg_weights:
return self.last_vg
objective = 0.0
gradient = CounterMap()
if verbose:
print "Calculating log probabilities and objective..."
# log_prob
log_probs = list()
for pos, (label, features) in enumerate(self.labeled_extracted_features):
log_probs.append(get_log_probs(features, weights, self.labels))
assert (
abs(sum(exp(log_probs[pos][label]) for label in self.labels) - 1.0) < 0.0001
), "Not a distribution: P[any | features] = %f" % (sum(exp(log_probs[pos][label]) for label in self.labels))
objective = -sum(log_prob[label] for (log_prob, (label, _)) in zip(log_probs, self.labeled_extracted_features))
if verbose:
print "Raw objective: %f" % objective
if verbose:
print "Calculating expected counts..."
expected_counts = get_expected_counts(self.labeled_extracted_features, self.labels, log_probs, CounterMap())
if verbose:
print "Calculating gradient..."
gradient = expected_counts - self.empirical_counts
if verbose:
print "Applying penalty"
# Apply a penalty (e.g. smooth the results)
if self.sigma:
penalty = 0.0
for label, feature_weights in gradient.iteritems():
for feature in feature_weights:
weight = weights[label][feature]
penalty += weight ** 2
gradient[label][feature] += weight / (self.sigma ** 2)
penalty /= 2 * self.sigma ** 2
objective += penalty
if verbose:
print "Penalized objective: %f" % objective
self.last_vg_weights = weights
self.last_vg = (objective, gradient)
return (objective, gradient)
def test_fast_slow_equal(self):
weights = CounterMap()
weights['cat'] = Counter(
(key, 1.0)
for key in ('fuzzy', 'claws', 'small', 'medium', 'large'))
weights['bear'] = Counter(
(key, 1.0)
for key in ('fuzzy', 'claws', 'small', 'medium', 'large'))
log_probs = [
maxent.get_log_probabilities(datum[1], weights, self.labels)
for datum in self.labeled_extracted_features
]
slow_expectation = maximumentropy.slow_expected_counts(
self.labeled_extracted_features, self.labels, log_probs)
fast_expectation = maxent.get_expected_counts(
self.labeled_extracted_features, self.labels, log_probs,
CounterMap())
self.assertEqual(slow_expectation, fast_expectation)
# And try again with different weights
weights['cat'] = Counter(
(key, 1.0) for key in ('fuzzy', 'claws', 'small', 'medium'))
weights['bear'] = Counter(
(key, 1.0) for key in ('fuzzy', 'claws', 'big'))
log_probs = [
maxent.get_log_probabilities(datum[1], weights, self.labels)
for datum in self.labeled_extracted_features
]
slow_expectation = maximumentropy.slow_expected_counts(
self.labeled_extracted_features, self.labels, log_probs)
fast_expectation = maxent.get_expected_counts(
self.labeled_extracted_features, self.labels, log_probs,
CounterMap())
self.assertEqual(slow_expectation, fast_expectation)
def value_and_gradient(self, weights, verbose=False):
if weights == self.last_vg_weights:
return self.last_vg
objective = 0.0
gradient = CounterMap()
if verbose: print "Calculating log probabilities and objective..."
# log_prob
log_probs = list()
for pos, (label,
features) in enumerate(self.labeled_extracted_features):
log_probs.append(get_log_probs(features, weights, self.labels))
assert abs(
sum(exp(log_probs[pos][label]) for label in self.labels) -
1.0) < 0.0001, "Not a distribution: P[any | features] = %f" % (
sum(exp(log_probs[pos][label]) for label in self.labels))
objective = -sum(log_prob[label] for (log_prob, (
label, _)) in zip(log_probs, self.labeled_extracted_features))
if verbose: print "Raw objective: %f" % objective
if verbose: print "Calculating expected counts..."
expected_counts = get_expected_counts(self.labeled_extracted_features,
self.labels, log_probs,
CounterMap())
if verbose: print "Calculating gradient..."
gradient = expected_counts - self.empirical_counts
if verbose: print "Applying penalty"
# Apply a penalty (e.g. smooth the results)
if self.sigma:
penalty = 0.0
for label, feature_weights in gradient.iteritems():
for feature in feature_weights:
weight = weights[label][feature]
penalty += weight**2
gradient[label][feature] += (weight / (self.sigma**2))
penalty /= 2 * self.sigma**2
objective += penalty
if verbose: print "Penalized objective: %f" % objective
self.last_vg_weights = weights
self.last_vg = (objective, gradient)
return (objective, gradient)
def test_fast_slow_equal(self):
weights = CounterMap()
weights['cat'] = Counter((key, 1.0) for key in ('fuzzy', 'claws', 'small', 'medium', 'large'))
weights['bear'] = Counter((key, 1.0) for key in ('fuzzy', 'claws', 'small', 'medium', 'large'))
log_probs = [maxent.get_log_probabilities(datum[1], weights, self.labels) for datum in self.labeled_extracted_features]
slow_expectation = maximumentropy.slow_expected_counts(self.labeled_extracted_features, self.labels, log_probs)
fast_expectation = maxent.get_expected_counts(self.labeled_extracted_features, self.labels, log_probs, CounterMap())
self.assertEqual(slow_expectation, fast_expectation)
# And try again with different weights
weights['cat'] = Counter((key, 1.0) for key in ('fuzzy', 'claws', 'small', 'medium'))
weights['bear'] = Counter((key, 1.0) for key in ('fuzzy', 'claws', 'big'))
log_probs = [maxent.get_log_probabilities(datum[1], weights, self.labels) for datum in self.labeled_extracted_features]
slow_expectation = maximumentropy.slow_expected_counts(self.labeled_extracted_features, self.labels, log_probs)
fast_expectation = maxent.get_expected_counts(self.labeled_extracted_features, self.labels, log_probs, CounterMap())
self.assertEqual(slow_expectation, fast_expectation)
from itertools import izip, repeat, chain
from maxent import get_log_probabilities, get_expected_counts
from countermap import CounterMap
from counter import Counter
def cnter(l):
return Counter(izip(l, repeat(1.0, len(l))))
training_data = (('cat', cnter(
('fuzzy', 'claws', 'small'))), ('bear', cnter(
('fuzzy', 'claws', 'big'))), ('cat', cnter(('claws', 'medium'))))
labels = set([label for label, _ in training_data])
features = set()
for _, counter in training_data:
features.update(set(counter.keys()))
weights = CounterMap()
log_probs = list()
for pos, (label, features) in enumerate(training_data):
log_probs.append(get_log_probabilities(features, weights, labels))
test = get_expected_counts(training_data, labels, log_probs, CounterMap())
print test
from itertools import izip, repeat, chain
from maxent import get_log_probabilities, get_expected_counts
from countermap import CounterMap
from counter import Counter
def cnter(l):
return Counter(izip(l, repeat(1.0, len(l))))
training_data = (('cat', cnter(('fuzzy', 'claws', 'small'))),
('bear', cnter(('fuzzy', 'claws', 'big'))),
('cat', cnter(('claws', 'medium'))))
labels = set([label for label, _ in training_data])
features = set()
for _, counter in training_data:
features.update(set(counter.keys()))
weights = CounterMap()
log_probs = list()
for pos, (label, features) in enumerate(training_data):
log_probs.append(get_log_probabilities(features, weights, labels))
test = get_expected_counts(training_data, labels, log_probs, CounterMap())
print test
