def test_inference(self):
# annotators agreeing, check that inferred correctness is either
# CCC or III
nclasses, nitems = 4, 50*8
# create random model (this is our ground truth model)
omega = np.ones((nclasses,)) / float(nclasses)
theta = np.ones((8,)) * 0.9999
true_model = ModelA(nclasses, theta, omega)
# create random data
annotations = true_model.generate_annotations(nitems)
posterior = true_model.infer_labels(annotations)
testing.assert_allclose(posterior.sum(1), 1., atol=1e-6, rtol=0.)
inferred = posterior.argmax(1)
expected = annotations.max(1)
testing.assert_equal(inferred, expected)
self.assertTrue(np.all(posterior[np.arange(nitems),inferred] > 0.999))
# at chance, disagreeing annotators: most accurate wins
omega = np.ones((nclasses,)) / float(nclasses)
theta = np.ones((8,))
theta[1:4] = np.array([0.9, 0.6, 0.5])
model = ModelA(nclasses, theta, omega)
data = np.array([[MV, 0, 1, 2, MV, MV, MV, MV,]])
posterior = model.infer_labels(data)
posterior = posterior[0]
self.assertTrue(posterior[0] > posterior[1]
> posterior[2] > posterior[3])
def test_log_likelihood(self):
# check that log likelihood is maximal at true parameters
nclasses, nitems = 3, 1500*8
# create random model and data (this is our ground truth model)
true_model = ModelA.create_initial_state(nclasses)
annotations = true_model.generate_annotations(nitems)
max_llhood = true_model.log_likelihood(annotations)
# perturb omega
for _ in range(20):
theta = true_model.theta
omega = np.random.normal(loc=true_model.omega, scale=0.1)
omega = np.clip(omega, 0.001, 0.999)
omega /= omega.sum()
model = ModelA(nclasses, omega=omega, theta=theta)
llhood = model.log_likelihood(annotations)
self.assertGreater(max_llhood, llhood)
# perturb theta
for _ in range(20):
omega = true_model.omega
theta = np.random.normal(loc=true_model.theta, scale=0.1)
theta = np.clip(theta, 0., 1.)
model = ModelA(nclasses, omega=omega, theta=theta)
llhood = model.log_likelihood(annotations)
self.assertGreater(max_llhood, llhood)
