예제 #1
0
class TestWorkers(unittest.TestCase):

    def setUp(self):
        self.client = TroiaClient(ADDRESS)

    def tearDown(self):
        self.client.delete()

    def test_AddGetEmptyWorkers(self):
        self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS)
        self.client.await_completion(self.client.post_compute())
        response = self.client.await_completion(self.client.get_workers())
        self.assertEqual('OK', response['status'])
        self.assertEqual([], response['result'])

    def test_AddGetWorkers_BeforeCompute(self):
        self.client.create(CATEGORIES)
        self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
        response = self.client.await_completion(self.client.get_workers())
        self.assertEqual(5, len(response['result']))
        for w in response['result']:
            self.assertEqual(5, w['value']['assigns'])

    def test_AddGetWorkers_AfterCompute(self):
        self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
        self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.client.await_completion(self.client.post_compute())

        #assigns check
        response = self.client.await_completion(self.client.get_workers())
        self.assertEqual(5, len(response['result']))
        for w in response['result']:
            self.assertEqual(5, w['value']['assigns'])
        #confusion matrices check
        response = self.client.await_completion(self.client.get_workers_confusion_matrix())

        exp = {
         u'worker1': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}], 
         u'worker3': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}], 
         u'worker2': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 0.3333}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.6666}], 
         u'worker5': [{u'to': u'p**n', u'from': u'p**n', u'value': 0.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 1.0}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}], 
         u'worker4': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}]}

        for w in response['result']:
            worker_name = w['workerName']
            for e1 in w['value']:
                exists = False
                for e2 in exp[worker_name]:
                    if e2['to'] == e1['to'] and e2['from'] == e1['from']:
                        exists = True
                        self.assertAlmostEqual(
                           e2['value'], 
                           e1['value'], 
                           2, 
                           "{}, from: {}, to: {}. expected:{}, was: {}".format(worker_name, e1['from'], e1['to'], e2['value'], e1['value']))
                self.assertTrue(exists)

    def test_GetWorkerInfo(self):
        self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
        self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
        self.client.await_completion(self.client.post_compute())

        response = self.client.await_completion(self.client.get_worker_info('worker1'))
        self.assertTrue(5, response['result']['value']['assigns'])
        self.assertEqual(2, response['result']['value']['goldTests'])
        self.assertEqual(1, response['result']['value']['correctGoldTests'])
        self.assertTrue('worker1', response['result']['workerName'])

    def test_GetWorkerAssigns(self):
        self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
        self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.client.await_completion(self.client.post_compute())

        response = self.client.await_completion(self.client.get_worker_assigns('worker1'))
        print response
        self.assertTrue(5, len(response['result']))
        for a in response['result']:
            self.assertTrue((a['worker'], a['object'], a['label']) in ASSIGNED_LABELS)
class TestQualitySensitivePayments(unittest.TestCase):
    def setUp(self):
        self.client = TroiaClient(ADDRESS)

    def tearDown(self):
        self.client.delete()

    def testQSP_ZeroWorkerQUality(self):
        assignedLabels = [
                            ('worker1', 'url1', 'cat2'),
                            ('worker1', 'url2', 'cat1'),
                            ('worker2', 'url1', 'cat1'),
                            ('worker2', 'url2', 'cat1'),
                            ('worker3', 'url1', 'cat1'),
                            ('worker3', 'url2', 'cat2')]
        goldLabels = [
                      ('url1', 'cat1'),
                      ('url2', 'cat2')
                      ]
        response = self.client.create(["cat1", "cat2"])
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_gold_data(goldLabels))
        self.assertEqual('OK', response['status'])

        response = self.client.post_compute()
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
        self.assertEqual('OK', response['status'])
        pprint(response)

        response = self.client.await_completion(self.client.get_workers_quality_payment())
        pprint(response)

    def testQSP_NegativeWorkerQuality(self):
        assignedLabels = [
                            ('worker1', 'url1', 'cat2'),
                            ('worker1', 'url2', 'cat1'),
                            ('worker1', 'url3', 'cat2'),
                            ('worker2', 'url1', 'cat1'),
                            ('worker2', 'url2', 'cat1'),
                            ('worker2', 'url3', 'cat1'),
                            ('worker3', 'url1', 'cat1'),
                            ('worker3', 'url2', 'cat2'),
                            ('worker3', 'url3', 'cat1')
                            ]
        goldLabels = [
                      ('url1', 'cat1'),
                      ('url2', 'cat2'),
                      ('url3', 'cat1')
                      ]
        response = self.client.create(["cat1", "cat2"], categoryPriors= [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.9}])
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_gold_data(goldLabels))
        self.assertEqual('OK', response['status'])

        response = self.client.post_compute()
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.get_workers_confusion_matrix())
        pprint(response)

        response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
        self.assertEqual('OK', response['status'])
        pprint(response)

        response = self.client.await_completion(self.client.get_workers_quality_payment())
        pprint(response)