def make_troia_job(job, gold_samples=[]): """ Creates Troia job corresponding to passed Job object. """ client = TroiaClient(settings.TROIA_HOST, None) job_id = make_job_id(job=job) TroiaJob.objects.create(job=job, troia_id=job_id) client.reset(job_id) client.load_categories(cost_matrix, job_id) client.load_gold_labels(gold_samples, job_id)
def run_troia_on_data(cost_matrix, gold_samples, workers_labels): troia_client = TroiaClient(TROIA_SERVER_URL, TIMEOUT) # we create client instance ID = TUTORIAL_JOB_ID troia_client.reset(ID) # just to make sure that we don't interfere with some old data troia_client.load_categories(cost_matrix, ID) # we send cost matrix that contains all labels / categories troia_client.load_gold_labels(gold_samples, ID) # send samples for which we know correct label # we could also do this that way: # for object_id, label in gold_samples: # troia_client.load_gold_label(object_id, label, ID) troia_client.load_worker_assigned_labels(workers_labels, ID) # send labels that worker assigned to objects # we could also do this that way: # for worker, object_id, label in workers_labels: # troia_client.load_worker_assigned_label(worker, object_id, label, ID) troia_client.compute_blocking(ITERATIONS, ID) # we start actual calculations results = troia_client.get_dawid_skene(ID) # we collect results and stats # pprint.pprint(results) # and just print them with some formatting objects_data = results['objects'].items() # extracting data related to objects object_label_probabilities = [(object_id, params['categoryProbability']) for object_id, params in objects_data] # we extract label distributions for objects print "Object label probabilities:" pprint.pprint(sorted(object_label_probabilities)) # and we print them # extracting data related to workers workers_summary = str(troia_client.print_worker_summary(True, ID)) print print "Workers summary:" print workers_summary
def run_troia_on_data(cost_matrix, gold_samples, workers_labels): troia_client = TroiaClient(TROIA_SERVER_URL, TIMEOUT) # we create client instance ID = TUTORIAL_JOB_ID troia_client.reset(ID) # just to make sure that we don't interfere with some old data troia_client.load_categories(cost_matrix, ID) # we send cost matrix that contains all labels / categories troia_client.load_gold_labels(gold_samples, ID) # send samples for which we know correct label # we could also do this that way: # for object_id, label in gold_samples: # troia_client.load_gold_label(object_id, label, ID) troia_client.load_worker_assigned_labels(workers_labels, ID) # send labels that worker assigned to objects # we could also do this that way: # for worker, object_id, label in workers_labels: # troia_client.load_worker_assigned_label(worker, object_id, label, ID) troia_client.compute_blocking(ITERATIONS, ID) # we start actual calculations results = troia_client.get_dawid_skene(ID) # we collect results and stats # pprint.pprint(results) # and just print them with some formatting objects_data = results['objects'].items() # extracting data related to objects object_label_probabilities = [(object_id, params['categoryProbability']) for object_id, params in objects_data] # we extract label distributions for objects print "Object label probabilities:" pprint.pprint(sorted(object_label_probabilities)) # and we print them # extracting data related to workers workers_summary = str(troia_client.print_worker_summary(True, ID)) print print "Workers summary:" print workers_summary
class TestClient(unittest.TestCase): base_url = "http://localhost:8080/GetAnotherLabel/rest/" labels = [u"dog", "cat", "pig"] objects = [u"o_dog_0", u"o_dog_1", "o_cat_0", "o_pig_0", "o_pig_1"] prior = .314 miss_class_cost = .141 def _misclass_cost(self, label): d = dict([(x, self.miss_class_cost) for x in self.labels if x != label]) d[label] = 0. return d def _labels(self): return [(l, self._misclass_cost(l)) for l in self.labels] assigned_labels = \ [("worker" + str(i), objects[0], "dog") for i in xrange(9)] + \ [("worker9", objects[0], "pig")] def setUp(self): self.tc = TroiaClient(self.base_url) self.ID = "42" self.tc.load_categories(self._labels(), self.ID) def tearDown(self): self.tc.reset(self.ID) def testSetUp(self): self.assertTrue(self.tc.exists(self.ID)) self.assertFalse(self.tc.exists(self.ID + "0")) def testPing(self): r = self.tc.ping() prefix = "\"processing request at: " self.assertTrue(r.startswith(prefix)) # I wanted to parse date but it its in some wired form so.. def testReset(self): r = self.tc.reset(self.ID) self.assertTrue(r.startswith("nullified the ds object")) self.assertFalse(self.tc.exists(self.ID)) def testExists(self): self.assertFalse(self.tc.exists(self.ID + "0")) self.assertTrue(self.tc.exists(self.ID)) def testLoadCategories(self): ID = self.ID + "AnotherTestID" self.tc.reset(ID) self.assertFalse(self.tc.exists(ID)) self.tc.load_categories(self._labels(), ID) self.assertTrue(self.tc.exists(ID)) r = self.tc.get_dawid_skene(ID) for k, v in r['categories'].iteritems(): self.assertTrue(k in self.labels) self.assertEqual(v['name'], k) for n, c in v['misclassification_cost'].iteritems(): print n, c, k, v if n == k: self.assertEqual(0., c) else: self.assertEqual(self.miss_class_cost, c) self.tc.reset(ID) def testLoadCosts(self): self.tc.load_costs([(u"dog", "cat", 0.511)], self.ID) r = self.tc.get_dawid_skene(self.ID) self.assertEqual( 0.511, r['categories'][u'dog']['misclassification_cost']['cat']) def testAssignLabel(self): job = self.assigned_labels[0] self.tc.load_worker_assigned_label(job[0], job[1], job[2], self.ID) r = self.tc.get_dawid_skene(self.ID) resp = r['workers'][job[0]]['labels'][0] print job sorted(job) sorted(resp.values()) self.assertEqual(sorted(job), sorted(resp.values())) def testAssignLabels(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) r = self.tc.get_dawid_skene(self.ID)['workers'] for job in self.assigned_labels: resp = r[job[0]]['labels'][0] self.assertEqual(sorted(job), sorted(resp.values())) # self.assertEqual(job, r[job['workerName']]['labels'][0]) def testGoldLabel(self): item = (self.objects[0], u"dog") self.tc.load_gold_label(item[0], item[1], self.ID) r = self.tc.get_dawid_skene(self.ID)['objects'] self.assertEqual(item[1], r[item[0]]["correctCategory"]) self.assertTrue(r[item[0]]["isGold"]) def testGoldLabels(self): gold_labels = [(self.objects[0], "dog"), (self.objects[-1], "pig")] self.tc.load_gold_labels(gold_labels, self.ID) r = self.tc.get_dawid_skene(self.ID)['objects'] for item in gold_labels: self.assertTrue(r[item[0]]["isGold"]) self.assertEqual(item[1], r[item[0]]["correctCategory"]) def testMajorityVote(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) self.tc.compute_blocking(3, self.ID) for _ in xrange(10): r = self.tc.majority_vote(self.objects[0], self.ID) print r # self.assertEqual(self.objects[0].split("_")[1], r) self.fail() def testMajorityVotes(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) self.tc.compute_blocking(3, self.ID) for _ in xrange(10): r = self.tc.majority_votes(self.ID) print r self.fail() def testComputeBlocking(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) r1 = self.tc.compute_blocking(1, self.ID) r3 = self.tc.compute_blocking(3, self.ID) print r1, r3 self.fail() def testPrintWorkerSummary(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) self.tc.compute_blocking(10, self.ID) r = self.tc.print_worker_summary(True, self.ID) self.assertEqual(10, r.count("Confusion Matrix:")) self.assertEqual(10, r.count("Number of Annotations: 1")) for l in self.assigned_labels: self.assertEqual(1, r.count("Worker: " + l[0])) def testPrintObjectsProbs(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) r = self.tc.print_objects_probs([], self.ID) print r self.fail() def testObjectProbs(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) r = self.tc.object_probs(self.objects[0], self.ID) from math import fsum self.assertAlmostEqual(1., fsum(r.values()), 10) def testPrintPriors(self): r = self.tc.print_priors(self.ID) for l in r.splitlines(): pass #self.assertIn(str(self.prior), l) # whe don't support priors def testClassPriors(self): r = self.tc.class_priors(self.ID) r = json.loads(r) for _, v in r.iteritems(): pass #self.assertEqual(self.prior, v) # whe don't support priors def testGetDawidSkene(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) self.tc.compute_blocking(3, self.ID) gold_labels = [{ "objectName": self.objects[0], "correctCategory": "dog" }, { "objectName": self.objects[-1], "correctCategory": "pig" }] self.tc.load_gold_labels(gold_labels, self.ID) self.tc.compute_blocking(3, self.ID) r = self.tc.get_dawid_skene(self.ID) for k in ["workers", "objects", "id", "fixedPriors", "categories"]: self.assertTrue(k in r.keys()) for l in self.assigned_labels: self.assertTrue(l[0] in r["workers"].keys())
class TestClient(unittest.TestCase): base_url = "http://localhost:8080/GetAnotherLabel/rest/" labels = [u"dog", "cat", "pig"] objects = [u"o_dog_0", u"o_dog_1", "o_cat_0", "o_pig_0", "o_pig_1"] prior = .314 miss_class_cost = .141 def _misclass_cost(self, label): d = dict([(x, self.miss_class_cost) for x in self.labels if x != label]) d[label] = 0. return d def _labels(self): return [(l, self._misclass_cost(l)) for l in self.labels] assigned_labels = \ [("worker" + str(i), objects[0], "dog") for i in xrange(9)] + \ [("worker9", objects[0], "pig")] def setUp(self): self.tc = TroiaClient(self.base_url) self.ID = "42" self.tc.load_categories(self._labels(), self.ID) def tearDown(self): self.tc.reset(self.ID) def testSetUp(self): self.assertTrue(self.tc.exists(self.ID)) self.assertFalse(self.tc.exists(self.ID + "0")) def testPing(self): r = self.tc.ping() prefix = "\"processing request at: " self.assertTrue(r.startswith(prefix)) # I wanted to parse date but it its in some wired form so.. def testReset(self): r = self.tc.reset(self.ID) self.assertTrue(r.startswith("nullified the ds object")) self.assertFalse(self.tc.exists(self.ID)) def testExists(self): self.assertFalse(self.tc.exists(self.ID + "0")) self.assertTrue(self.tc.exists(self.ID)) def testLoadCategories(self): ID = self.ID + "AnotherTestID" self.tc.reset(ID) self.assertFalse(self.tc.exists(ID)) self.tc.load_categories(self._labels(), ID) self.assertTrue(self.tc.exists(ID)) r = self.tc.get_dawid_skene(ID) for k, v in r['categories'].iteritems(): self.assertTrue(k in self.labels) self.assertEqual(v['name'], k) for n, c in v['misclassification_cost'].iteritems(): print n, c, k, v if n == k: self.assertEqual(0., c) else: self.assertEqual(self.miss_class_cost, c) self.tc.reset(ID) def testLoadCosts(self): self.tc.load_costs( [(u"dog", "cat", 0.511)], self.ID) r = self.tc.get_dawid_skene(self.ID) self.assertEqual(0.511, r['categories'][u'dog']['misclassification_cost']['cat']) def testAssignLabel(self): job = self.assigned_labels[0] self.tc.load_worker_assigned_label(job[0], job[1], job[2], self.ID) r = self.tc.get_dawid_skene(self.ID) resp = r['workers'][job[0]]['labels'][0] print job sorted(job) sorted(resp.values()) self.assertEqual(sorted(job), sorted(resp.values())) def testAssignLabels(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) r = self.tc.get_dawid_skene(self.ID)['workers'] for job in self.assigned_labels: resp = r[job[0]]['labels'][0] self.assertEqual(sorted(job), sorted(resp.values())) # self.assertEqual(job, r[job['workerName']]['labels'][0]) def testGoldLabel(self): item = (self.objects[0], u"dog") self.tc.load_gold_label(item[0], item[1], self.ID) r = self.tc.get_dawid_skene(self.ID)['objects'] self.assertEqual(item[1], r[item[0]]["correctCategory"]) self.assertTrue(r[item[0]]["isGold"]) def testGoldLabels(self): gold_labels = [(self.objects[0], "dog"), (self.objects[-1], "pig")] self.tc.load_gold_labels(gold_labels, self.ID) r = self.tc.get_dawid_skene(self.ID)['objects'] for item in gold_labels: self.assertTrue(r[item[0]]["isGold"]) self.assertEqual(item[1], r[item[0]]["correctCategory"]) def testMajorityVote(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) self.tc.compute_blocking(3, self.ID) for _ in xrange(10): r = self.tc.majority_vote(self.objects[0], self.ID) print r # self.assertEqual(self.objects[0].split("_")[1], r) self.fail() def testMajorityVotes(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) self.tc.compute_blocking(3, self.ID) for _ in xrange(10): r = self.tc.majority_votes(self.ID) print r self.fail() def testComputeBlocking(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) r1 = self.tc.compute_blocking(1, self.ID) r3 = self.tc.compute_blocking(3, self.ID) print r1, r3 self.fail() def testPrintWorkerSummary(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) self.tc.compute_blocking(10, self.ID) r = self.tc.print_worker_summary(True, self.ID) self.assertEqual(10, r.count("Confusion Matrix:")) self.assertEqual(10, r.count("Number of Annotations: 1")) for l in self.assigned_labels: self.assertEqual(1, r.count("Worker: " + l[0])) def testPrintObjectsProbs(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) r = self.tc.print_objects_probs([], self.ID) print r self.fail() def testObjectProbs(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) r = self.tc.object_probs(self.objects[0], self.ID) from math import fsum self.assertAlmostEqual(1., fsum(r.values()), 10) def testPrintPriors(self): r = self.tc.print_priors(self.ID) for l in r.splitlines(): pass #self.assertIn(str(self.prior), l) # whe don't support priors def testClassPriors(self): r = self.tc.class_priors(self.ID) r = json.loads(r) for _, v in r.iteritems(): pass #self.assertEqual(self.prior, v) # whe don't support priors def testGetDawidSkene(self): self.tc.load_worker_assigned_labels(self.assigned_labels, self.ID) self.tc.compute_blocking(3, self.ID) gold_labels = [{"objectName": self.objects[0], "correctCategory": "dog"}, {"objectName": self.objects[-1], "correctCategory": "pig"}] self.tc.load_gold_labels(gold_labels, self.ID) self.tc.compute_blocking(3, self.ID) r = self.tc.get_dawid_skene(self.ID) for k in ["workers", "objects", "id", "fixedPriors", "categories"]: self.assertTrue(k in r.keys()) for l in self.assigned_labels: self.assertTrue(l[0] in r["workers"].keys())