def test_log_likelihood(self):
# check that log likelihood is maximal at true parameters
nclasses, nannotators, nitems = 3, 5, 1000
# create random model and data (this is our ground truth model)
true_model = ModelBt.create_initial_state(nclasses, nannotators)
annotations = true_model.generate_annotations(nitems)
max_llhood = true_model.log_likelihood(annotations)
# perturb gamma
for _ in xrange(20):
theta = true_model.theta
gamma = np.random.normal(loc=true_model.gamma, scale=0.1)
gamma = np.clip(gamma, 0., 1.)
gamma /= gamma.sum()
model = ModelBt(nclasses, nannotators, gamma, theta)
llhood = model.log_likelihood(annotations)
self.assertGreater(max_llhood, llhood)
# perturb theta
for _ in xrange(20):
gamma = true_model.gamma
theta = np.random.normal(loc=true_model.theta, scale=0.1)
theta = np.clip(theta, 0., 1.)
model = ModelBt(nclasses, nannotators, gamma, theta)
llhood = model.log_likelihood(annotations)
self.assertGreater(max_llhood, llhood)
def test_log_likelihood(self):
# check that log likelihood is maximal at true parameters
nclasses, nannotators, nitems = 3, 5, 1000
# create random model and data (this is our ground truth model)
true_model = ModelBt.create_initial_state(nclasses, nannotators)
annotations = true_model.generate_annotations(nitems)
max_llhood = true_model.log_likelihood(annotations)
# perturb gamma
for _ in range(20):
theta = true_model.theta
gamma = np.random.normal(loc=true_model.gamma, scale=0.1)
gamma = np.clip(gamma, 0., 1.)
gamma /= gamma.sum()
model = ModelBt(nclasses, nannotators, gamma, theta)
llhood = model.log_likelihood(annotations)
self.assertGreater(max_llhood, llhood)
# perturb theta
for _ in range(20):
gamma = true_model.gamma
theta = np.random.normal(loc=true_model.theta, scale=0.1)
theta = np.clip(theta, 0., 1.)
model = ModelBt(nclasses, nannotators, gamma, theta)
llhood = model.log_likelihood(annotations)
self.assertGreater(max_llhood, llhood)
def test_log_likelihood_loop_design(self):
# behavior: the log likelihood of the new class should match the one
# of the more specialized class
nclasses, nannotators, nitems = 4, 8, 100
# create specialized model, draw data
true_model = ModelBtLoopDesign.create_initial_state(nclasses)
annotations = true_model.generate_annotations(nitems)
expect = true_model.log_likelihood(annotations)
model = ModelBt(nclasses, nannotators,
gamma=true_model.gamma, theta=true_model.theta)
llhood = model.log_likelihood(annotations)
np.testing.assert_almost_equal(llhood, expect, 10)
def test_log_likelihood_loop_design(self):
# behavior: the log likelihood of the new class should match the one
# of the more specialized class
nclasses, nannotators, nitems = 4, 8, 100
# create specialized model, draw data
true_model = ModelBtLoopDesign.create_initial_state(nclasses)
annotations = true_model.generate_annotations(nitems)
expect = true_model.log_likelihood(annotations)
model = ModelBt(nclasses,
nannotators,
gamma=true_model.gamma,
theta=true_model.theta)
llhood = model.log_likelihood(annotations)
np.testing.assert_almost_equal(llhood, expect, 10)