class TestWorkers(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def tearDown(self):
self.client.delete()
def test_AddGetEmptyWorkers(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS)
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_workers())
self.assertEqual('OK', response['status'])
self.assertEqual([], response['result'])
def test_AddGetWorkers_BeforeCompute(self):
self.client.create(CATEGORIES)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
response = self.client.await_completion(self.client.get_workers())
self.assertEqual(5, len(response['result']))
for w in response['result']:
self.assertEqual(5, w['value']['assigns'])
def test_AddGetWorkers_AfterCompute(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_compute())
#assigns check
response = self.client.await_completion(self.client.get_workers())
self.assertEqual(5, len(response['result']))
for w in response['result']:
self.assertEqual(5, w['value']['assigns'])
#confusion matrices check
response = self.client.await_completion(self.client.get_workers_confusion_matrix())
exp = {
u'worker1': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}],
u'worker3': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}],
u'worker2': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.},
{u'to': u'p**n', u'from': u'notporn', u'value': 0.3333}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.6666}],
u'worker5': [{u'to': u'p**n', u'from': u'p**n', u'value': 0.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 1.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}],
u'worker4': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}]}
for w in response['result']:
worker_name = w['workerName']
for e1 in w['value']:
exists = False
for e2 in exp[worker_name]:
if e2['to'] == e1['to'] and e2['from'] == e1['from']:
exists = True
self.assertAlmostEqual(
e2['value'],
e1['value'],
2,
"{}, from: {}, to: {}. expected:{}, was: {}".format(worker_name, e1['from'], e1['to'], e2['value'], e1['value']))
self.assertTrue(exists)
def test_GetWorkerInfo(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_worker_info('worker1'))
self.assertTrue(5, response['result']['value']['assigns'])
self.assertEqual(2, response['result']['value']['goldTests'])
self.assertEqual(1, response['result']['value']['correctGoldTests'])
self.assertTrue('worker1', response['result']['workerName'])
def test_GetWorkerAssigns(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_worker_assigns('worker1'))
print response
self.assertTrue(5, len(response['result']))
for a in response['result']:
self.assertTrue((a['worker'], a['object'], a['label']) in ASSIGNED_LABELS)
class TestQualitySensitivePayments(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def tearDown(self):
self.client.delete()
def testQSP_ZeroWorkerQUality(self):
assignedLabels = [
('worker1', 'url1', 'cat2'),
('worker1', 'url2', 'cat1'),
('worker2', 'url1', 'cat1'),
('worker2', 'url2', 'cat1'),
('worker3', 'url1', 'cat1'),
('worker3', 'url2', 'cat2')]
goldLabels = [
('url1', 'cat1'),
('url2', 'cat2')
]
response = self.client.create(["cat1", "cat2"])
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_gold_data(goldLabels))
self.assertEqual('OK', response['status'])
response = self.client.post_compute()
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
self.assertEqual('OK', response['status'])
pprint(response)
response = self.client.await_completion(self.client.get_workers_quality_payment())
pprint(response)
def testQSP_NegativeWorkerQuality(self):
assignedLabels = [
('worker1', 'url1', 'cat2'),
('worker1', 'url2', 'cat1'),
('worker1', 'url3', 'cat2'),
('worker2', 'url1', 'cat1'),
('worker2', 'url2', 'cat1'),
('worker2', 'url3', 'cat1'),
('worker3', 'url1', 'cat1'),
('worker3', 'url2', 'cat2'),
('worker3', 'url3', 'cat1')
]
goldLabels = [
('url1', 'cat1'),
('url2', 'cat2'),
('url3', 'cat1')
]
response = self.client.create(["cat1", "cat2"], categoryPriors= [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.9}])
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_gold_data(goldLabels))
self.assertEqual('OK', response['status'])
response = self.client.post_compute()
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.get_workers_confusion_matrix())
pprint(response)
response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
self.assertEqual('OK', response['status'])
pprint(response)
response = self.client.await_completion(self.client.get_workers_quality_payment())
pprint(response)