class TestGoldLabels(unittest.TestCase):
    def setUp(self):
        self.client = TroiaClient(ADDRESS)
        response = self.client.create(CATEGORIES)
        self.assertEqual("OK", response["status"])

    def tearDown(self):
        self.client.delete()

    def _test_method(self, goldLabels):
        response = self.client.await_completion(self.client.post_gold_data(goldLabels))
        self.assertEqual("OK", response["status"])

        # get the gold labels
        response = self.client.await_completion(self.client.get_gold_data())
        self.assertEqual("OK", response["status"])
        result = response["result"]
        self.assertEqual(len(goldLabels), len(result))

        results = [tuple(receivedLabel.values()) for receivedLabel in response["result"]]
        for label in goldLabels:
            self.assertTrue(label in results)

    def test_AddGetEmptyGoldLabels(self):
        self._test_method([])

    def test_AddGetGoldLabel_LongLabelName(self):
        response = self.client.await_completion(
            self.client.post_gold_data(
                [
                    (
                        "sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs",
                        "notporn",
                    )
                ]
            )
        )
        self.assertEqual("ERROR", response["status"])
        self.assertEqual("Internal error: Object name should be shorter than 100 chars", response["result"])

    def test_AddGetGoldLabel_PrintableASCII_RegularChars(self):
        self._test_method([("url1", "notporn")])

    def test_AddGetGoldLabel_PrintableASCII_SpecialChars(self):
        self._test_method([("~!@#^&*()_+=-[]{}:<>,./", "notporn")])

    def test_AddGetGoldLabel_ExtendedASCIIChars(self):
        self._test_method([(u"™ž¤©", "notporn")])

    def test_AddGetGoldLabel_UnicodeChars(self):
        self._test_method([(u"ૉେஇ", "notporn")])
class TestCommandStatus(unittest.TestCase):

    def setUp(self):
        self.tc = TroiaClient(ADDRESS)

    def tearDown(self):
        self.tc.delete()

    def test_GetRedirectData(self):
        self.tc.create(CATEGORIES)
        assigend_labels_response = self.tc.get_assigned_labels()
        time.sleep(0.5)
        self.assertLessEqual(self.tc.get_status(assigend_labels_response['redirect'])['result'], [])
        self.tc.await_completion(self.tc.post_assigned_labels(ASSIGNED_LABELS))
        self.assertEqual(len(ASSIGNED_LABELS), len(self.tc.await_completion(self.tc.get_assigned_labels())['result']))
        #still empty
        self.assertListEqual(self.tc.get_status(assigend_labels_response['redirect'])['result'], [])
class TestEvaluationLabels(unittest.TestCase):

    def setUp(self):
        self.client = TroiaClient(ADDRESS)
        response = self.client.create(CATEGORIES)
        self.assertEqual('OK', response['status'])

    def tearDown(self):
        self.client.delete()

    def _test_method(self, eval_data):
        response = self.client.await_completion(self.client.post_evaluation_objects(eval_data))
        self.assertEqual('OK', response['status'])

        #get the unassigned labels
        response = self.client.await_completion(self.client.get_evaluation_objects())

        self.assertEqual('OK', response['status'])
        result = response['result']
        self.assertEqual(len(eval_data), len(result))

        results = [(evaluationLabel['name'], evaluationLabel['evaluationLabel']) for evaluationLabel in response['result']]

        for evalLabel in eval_data:
            self.assertTrue(evalLabel in results)

    def test_AddGetEmptyEvaluationLabels(self):
        self._test_method([])

    def test_AddGetEvaluationLabels_PrintableASCII_RegularChars(self):
        self._test_method(EVALUATION_DATA)

    def test_AddGetEvaluationLabel_PrintableASCII_SpecialChars(self):
        self._test_method([('~!@%#^&*()_+=-[]{}:<>,./', 'notporn')])

    def test_AddGetEvaluationLabel_ExtendedASCIIChars(self):
        self._test_method([(u'™ž¤©', 'notporn')])

    def test_AddGetEvaluationLabel_UnicodeChars(self):
        self._test_method([(u'ૉେஇ', 'notporn')])

    def test_AddGetEvaluationLabel_LongLabelNames(self):
        response = self.client.await_completion(self.client.post_evaluation_objects([('sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs', 'notporn')]))
        self.assertEqual('ERROR', response['status'])
        self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result'])
class TestPrediction(unittest.TestCase):

    def setUp(self):
        self.client = TroiaClient(ADDRESS)
        self.client.create(CATEGORIES, costMatrix=COST_MATRIX, categoryPriors=CATEGORY_PRIORS, algorithm=self.algorithm)
        self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.client.await_completion(self.client.post_evaluation_objects(EVALUATION_DATA))
        self.client.await_completion(self.client.post_compute())

    def tearDown(self):
        self.client.delete()

    def _getObjectsPrediction(self, method, expectedResults):
        response = self.client.await_completion(self.client.get_objects_prediction(method))
        self.assertEqual('OK', response['status'])
        for categories in response['result']:
            self.assertEqual(expectedResults[categories['objectName']], categories['categoryName'])

    def _getEstimatedObjectsCost(self, costMethod, expectedCosts):
        response = self.client.await_completion(self.client.get_estimated_objects_cost(costMethod))
        self.assertEqual('OK', response['status'])
        for cost in response['result']:
            self.assertTrue(abs(expectedCosts[cost['objectName']] - cost['value']) / 100 < TOLERANCE)

    def _getEvaluatedObjectsCost(self, labelChoosingMethod, expectedResults):
        response = self.client.await_completion(self.client.get_evaluated_objects_cost(labelChoosingMethod))
        self.assertEqual('OK', response['status'])
        for item in response['result']:
            self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE)

    def _getEstimatedObjectsQuality(self, costAlgorithm, expectedDataQuality):
        response = self.client.await_completion(self.client.get_estimated_objects_quality(costAlgorithm))
        self.assertEqual('OK', response['status'])
        for dataQuality in response['result']:
            self.assertTrue(abs(expectedDataQuality[dataQuality['objectName']] - dataQuality['value']) / 100 < TOLERANCE)

    def _getEvaluatedObjectsQuality(self, labelChoosingMethod, expectedResults):
        response = self.client.await_completion(self.client.get_evaluated_objects_quality(labelChoosingMethod))
        self.assertEqual('OK', response['status'])
        for item in response['result']:
            self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE)

    def _getEstimatedObjectsCostSummary (self, expectedObjectsCosts):
        response = self.client.await_completion(self.client.get_objects_cost_estimated_summary())
        self.assertEqual('OK', response['status'])
        for k, v in expectedObjectsCosts.items():
            self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE)

    def _getEvaluatedObjectsCostSummary (self, expectedObjectsCosts):
        response = self.client.await_completion(self.client.get_objects_cost_evaluated_summary())
        self.assertEqual('OK', response['status'])
        for k, v in expectedObjectsCosts.items():
            self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE)

    def _getEstimatedObjectsQualitySummary(self, expectedDataQuality):
        response = self.client.await_completion(self.client.get_objects_quality_estimated_summary())
        self.assertEqual('OK', response['status'])
        for k, v in expectedDataQuality.items():
            self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE)

    def _getEvaluatedObjectsQualitySummary(self, expectedDataQuality):
        response = self.client.await_completion(self.client.get_objects_quality_evaluated_summary())
        self.assertEqual('OK', response['status'])
        for k, v in expectedDataQuality.items():
            self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE)

    def _getEstimatedWorkerCost(self, costAlgorithm, expectedWorkerCost):
        response = self.client.await_completion(self.client.get_estimated_workers_cost(costAlgorithm))
        self.assertEqual('OK', response['status'])
        for workerCost in response['result']:
            self.assertTrue(abs(expectedWorkerCost[workerCost['workerName']] - workerCost['value']) / 100 < TOLERANCE)

    def _getEstimatedWorkerQuality(self, costAlgorithm, expectedWorkerQuality):
        response = self.client.await_completion(self.client.get_estimated_workers_quality(costAlgorithm))
        self.assertEqual('OK', response['status'])
        for workerQuality in response['result']:
            self.assertTrue(abs(expectedWorkerQuality[workerQuality['workerName']] - workerQuality['value']) / 100 < TOLERANCE)

    def _getEvaluatedWorkerQuality(self, costAlgorithm, expectedResults):
        response = self.client.await_completion(self.client.get_evaluated_workers_quality(costAlgorithm))
        self.assertEqual('OK', response['status'])
        for item in response['result']:
            self.assertTrue(abs(expectedResults[item['workerName']] - item['value']) / 100 < TOLERANCE)

    def _getWorkersQualityEstimatedSummary(self, expectedWorkerQuality):
        response = self.client.await_completion(self.client.get_workers_quality_estimated_summary())
        self.assertEqual('OK', response['status'])
        for k, v in expectedWorkerQuality.items():
            self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE)

    def _getWorkersQualityEvaluatedSummary(self, expectedWorkerQuality):
        response = self.client.await_completion(self.client.get_workers_quality_evaluated_summary())
        self.assertEqual('OK', response['status'])
        for k, v in expectedWorkerQuality.items():
            self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE)

    def _getCategoryProbability(self, expectedProbabilities):
        for object in set((x[1] for x in ASSIGNED_LABELS)):
            response = self.client.await_completion(self.client.get_probability_distribution(object))
            self.assertEqual('OK', response['status'])
            self.assertEqual(expectedProbabilities[object], response['result'])
class TestCategories(unittest.TestCase):

        def setUp(self):
            self.client = TroiaClient(ADDRESS)

        def tearDown(self):
            self.client.delete()

        def _test_method(self, categories):
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_categories())
            self.assertEqual('OK', response['status'])
            for category in categories:
                self.assertTrue(category in response['result'])

        def _create_priors(self, categories, priors):
            return [{'categoryName': c, "value": p} for c, p in zip(categories, priors)]

        def test_AddGetCategories_LongCategoryNames(self):
            categories = [u'hjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdh', u'hjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhsdsd']
            response = self.client.create(categories)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Category names should be shorter than 50 chars', response['result'])

        def test_AddGetCategories_PrintableASCII_SpecialChars(self):
            categories = [u'!@#$,.{}[]', u'2ndCategory']
            self._test_method(categories)

        def test_AddGetCategories_ExtendedASCIIChars(self):
            categories = [u'œŒ', u'ÀÆË™ž¤©']
            self._test_method(categories)

        def test_AddGetCategories_UnicodeChars(self):
            categories = [u'ૉେஇΨҖӖմ؂څ', u'ూഹܬआਖ਼']
            self._test_method(categories)

        def test_CreateJobWithoutCategoryPriors_GetPriors(self):
            categories = [u'category1', u'category2']
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_categories())
            self.assertEqual('OK', response['status'])
            for category in categories:
                self.assertTrue(category in response['result'])

        def test_CreateJobWithCategoryPriors_GetPriors(self):
            categories = [u'category1', u'category2']
            priors = [0.3, 0.7]
            response = self.client.create(categories, categoryPriors=self._create_priors(categories, priors))
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_categories())
            self.assertEqual('OK', response['status'])
            for category in categories:
                self.assertTrue(category in response['result'])

        def test_CreateJobWithoutCostMatrix_2Categories_GetCostMatrix(self):
            categories = [u'category1', u'category2']
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_cost_matrix())
            self.assertEqual('OK', response['status'])

            #check that the default 0-1 cost matrix is created
            for matrix_value in response['result']:
                self.assertEqual(0.0 if matrix_value['from'] == matrix_value['to'] else 1.0, matrix_value['value'])

        def test_CreateJobWithoutCostMatrix_3Categories_GetCostMatrix(self):
            categories = [u'category1', u'category2', u'category3']
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_cost_matrix())
            self.assertEqual('OK', response['status'])

            #check that the default 0-1 cost matrix is created
            for matrix_value in response['result']:
                self.assertEqual(0.0 if matrix_value['from'] == matrix_value['to'] else 1.0, matrix_value['value'])

        def test_CreateJobWithCostMatrix_GetCostMatrix(self):
            categories = [u'category1', u'category2', u'category3']
            cMatrix = [{"from": "category1", "to": "category1", "value": 0.0}, {"from": "category1", "to": "category2", "value": 0.5}, {"from": "category1", "to": "category3", "value": 0.5},
                       {"from": "category2", "to": "category1", "value": 0.5}, {"from": "category2", "to": "category2", "value": 0.0}, {"from": "category2", "to": "category3", "value": 0.5},
                       {"from": "category3", "to": "category1", "value": 0.5}, {"from": "category3", "to": "category2", "value": 0.5}, {"from": "category3", "to": "category3", "value": 0.0}]
            response = self.client.create(categories, costMatrix=cMatrix)
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_cost_matrix())
            self.assertEqual('OK', response['status'])
            for matrix_value in response['result']:
                self.assertEqual(0.0 if matrix_value['from'] == matrix_value['to'] else 0.5, matrix_value['value'])
class TestUnassignedLabels(unittest.TestCase):
    def setUp(self):
        self.client = TroiaClient(ADDRESS)

    def get_priors(self, categories):
        return [{'categoryName': c, "value": 1. / len(categories)} for c in categories]

    def tearDown(self):
        self.client.delete()

    def _test_method(self, categories, priors, unassignedLabels, expectedProbabilities):
        response = self.client.create(categories, categoryPriors=priors, algorithm="BMV")
        self.assertEqual('OK', response['status'])

        #post the unassigned labels
        response = self.client.await_completion(self.client.post_objects(unassignedLabels))
        self.assertEqual('OK', response['status'])

        #get the unassigned labels
        response = self.client.await_completion(self.client.get_objects())
        self.assertEqual('OK', response['status'])
        self.assertEqual(len(unassignedLabels), len(response['result']))
        if (unassignedLabels):
            result = response['result'][0]
            self.assertEqual(unassignedLabels[0], result['name'])

    def test_AddGetObjects_EmptyLabels(self):
        self._test_method(CATEGORIES, CATEGORY_PRIORS, [], [])

    def test_AddGetUnassignedLabels_LongLabelNames(self):
        categories = ["category1", "category2"]
        priors = [{"categoryName": "category1", "value": 0.0000000001}, {"categoryName": "category2", "value": 0.9999999999}]
        unassignedLabels = ["sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs"]
        expectedProbabilities = [('category1', 0.5), ('category2', 0.5)]
        response = self.client.create(categories, categoryPriors=priors, algorithm="BMV")
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_objects(unassignedLabels))
        self.assertEqual('ERROR', response['status'])
        self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result'])


    def test_AddGetUnassignedLabels_PrintableASCII_RegularChars(self):
        categories = ["category1", "category2", "category3", "category4"]
        unassignedLabels = ["testObject1"]
        expectedProbabilities = [('category1', 0.25),
                                 ('category2', 0.25),
                                 ('category3', 0.25),
                                 ('category4', 0.25)]
        self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)

    def test_AddGetUnassignedLabels_PrintableASCII_SpecialChars(self):
        categories = ["category1", "category2", "category3"]
        unassignedLabels = ["~!@#%^&*()_+=-[]{}:<>,./"]
        expectedProbabilities = [('category1', 0.3333333333333333),
                                 ('category2', 0.3333333333333333),
                                 ('category3', 0.3333333333333333)]
        self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)

    def test_AddGetUnassignedLabels_ExtendedASCIIChars(self):
        categories = ["category1", "category2"]
        unassignedLabels = [u"ëñµ¼Úæ"]
        expectedProbabilities = [('category1', 0.5), ('category2', 0.5)]
        self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)

    def test_AddGetUnassignedLabels_UnicodeChars(self):
        categories = ["category1", "category2"]
        unassignedLabels = [u"ూഹܬआਖ਼"]
        expectedProbabilities = [('category1', 0.5), ('category2', 0.5)]
        self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)

    def test_GetObjectAssigns(self):
        response = self.client.create(CATEGORIES)
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.assertEqual('OK', response['status'])

        #post the unassigned label
        unassignedLabel = [u"ూഹܬआਖ਼"]
        response = self.client.await_completion(self.client.post_objects(unassignedLabel))
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.get_objects())
        self.assertEqual('OK', response['status'])
        result = response['result']
        self.assertEqual(6, len(result))
        results = []
        for object in result:
            if object['name'] not in unassignedLabel:
                assigns = self.client.await_completion(self.client.get_object_assigns(object['name']))['result']
                for a in assigns:
                    results.append((a['worker'], a['object'], a['label']))
        for label in ASSIGNED_LABELS:
            self.assertTrue(label in results)

    def test_GetObjectInfo(self):
        response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS)
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_assigned_labels([('worker1', 'url1', 'p**n'), ('worker1', 'url2', 'p**n')]))
        self.assertEqual('OK', response['status'])

        #post the unassigned label
        objects_without_assigns = ["newUnassignedLabel"]
        response = self.client.await_completion(self.client.post_objects(objects_without_assigns))
        self.assertEqual('OK', response['status'])

        #get labels
        response = self.client.await_completion(self.client.get_objects())
        self.assertEqual('OK', response['status'])
        result = response['result']
        self.assertEqual(3, len(result))
        self.client.await_completion(self.client.post_compute())
        for label in result:
            response = self.client.await_completion(self.client.get_probability_distribution(label['name']))
            dist = response['result'][0]
            self.assertAlmostEqual(dist['value'], 0.5 if label['name'] in objects_without_assigns else 0.9 if dist['categoryName'] == 'p**n' else 0.1)
class TestWorkers(unittest.TestCase):

    def setUp(self):
        self.client = TroiaClient(ADDRESS)

    def tearDown(self):
        self.client.delete()

    def test_AddGetEmptyWorkers(self):
        self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS)
        self.client.await_completion(self.client.post_compute())
        response = self.client.await_completion(self.client.get_workers())
        self.assertEqual('OK', response['status'])
        self.assertEqual([], response['result'])

    def test_AddGetWorkers_BeforeCompute(self):
        self.client.create(CATEGORIES)
        self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
        response = self.client.await_completion(self.client.get_workers())
        self.assertEqual(5, len(response['result']))
        for w in response['result']:
            self.assertEqual(5, w['value']['assigns'])

    def test_AddGetWorkers_AfterCompute(self):
        self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
        self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.client.await_completion(self.client.post_compute())

        #assigns check
        response = self.client.await_completion(self.client.get_workers())
        self.assertEqual(5, len(response['result']))
        for w in response['result']:
            self.assertEqual(5, w['value']['assigns'])
        #confusion matrices check
        response = self.client.await_completion(self.client.get_workers_confusion_matrix())

        exp = {
         u'worker1': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}], 
         u'worker3': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}], 
         u'worker2': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 0.3333}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.6666}], 
         u'worker5': [{u'to': u'p**n', u'from': u'p**n', u'value': 0.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 1.0}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}], 
         u'worker4': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, 
                      {u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}]}

        for w in response['result']:
            worker_name = w['workerName']
            for e1 in w['value']:
                exists = False
                for e2 in exp[worker_name]:
                    if e2['to'] == e1['to'] and e2['from'] == e1['from']:
                        exists = True
                        self.assertAlmostEqual(
                           e2['value'], 
                           e1['value'], 
                           2, 
                           "{}, from: {}, to: {}. expected:{}, was: {}".format(worker_name, e1['from'], e1['to'], e2['value'], e1['value']))
                self.assertTrue(exists)

    def test_GetWorkerInfo(self):
        self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
        self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
        self.client.await_completion(self.client.post_compute())

        response = self.client.await_completion(self.client.get_worker_info('worker1'))
        self.assertTrue(5, response['result']['value']['assigns'])
        self.assertEqual(2, response['result']['value']['goldTests'])
        self.assertEqual(1, response['result']['value']['correctGoldTests'])
        self.assertTrue('worker1', response['result']['workerName'])

    def test_GetWorkerAssigns(self):
        self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
        self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
        self.client.await_completion(self.client.post_compute())

        response = self.client.await_completion(self.client.get_worker_assigns('worker1'))
        print response
        self.assertTrue(5, len(response['result']))
        for a in response['result']:
            self.assertTrue((a['worker'], a['object'], a['label']) in ASSIGNED_LABELS)
class TestCachedScheduler(unittest.TestCase):

    def setUp(self):
        self.client = TroiaClient(ADDRESS)
        self.scheduler = 'CachedScheduler'
        self.utils = TestUtils()

    def tearDown(self):
        self.client.delete()

    def _createTestPrereq(self, algorithm, scheduler, calculator, assigns, categories = CATEGORIES, categoryPriors = CATEGORY_PRIORS, costMatrix = COST_MATRIX):
        response = self.client.create(
            categories,
            categoryPriors=categoryPriors,
            algorithm=algorithm,
            scheduler=scheduler,
            prioritycalculator=calculator,
            costMatrix=costMatrix
            )

        self.assertEqual('OK', response['status'])
        self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(assigns))['status'])
        self.assertEqual('OK', self.client.await_completion(self.client.post_compute())['status'])

    def getObjectCountsList(self, assigns, reverse=False, excludedObjectsList=None):
        #create the dictionary containing the objects and the associated no of assigns
        labelsDict = {}
        for l in assigns:
                if l[1] in labelsDict.keys():
                    labelsDict[l[1]] += 1
                else:
                    labelsDict[l[1]] = 1

        #sort the objects ascending, based on the no of labels 
        sortedList = sorted(labelsDict.items(), key=itemgetter(1), reverse=reverse)
        if (excludedObjectsList):
            for value in sortedList:
                if value[0] in excludedObjectsList:
                    sortedList.remove(value)
        return sortedList

    def getObjectCostsList(self):
        response = self.client.await_completion(self.client.get_estimated_objects_cost("ExpectedCost"))
        self.assertEquals("OK", response['status'])
        objectCosts = {}
        for result in response['result']:
            objectCosts[result['objectName']] = result['value']

        #sort the objects descending, based on cost 
        objectCostList = sorted(objectCosts.items(), key=itemgetter(1), reverse=True)
        return objectCostList

    def getAssignedLabels(self):
        response = self.client.await_completion(self.client.get_assigned_labels())
        self.assertEqual('OK', response['status'])
        assignedLabels = [(l['worker'], l['object'], l['label']) for l in response['result']]
        return assignedLabels

    def _runScheduler(self, workerId=None):
        if (workerId):
            return self.client.get_next_worker_object(workerId)
        else:
            return self.client.get_next_object()

    def _runTestMethod(self, calculator, expectedObjectList, newAssign, workerId=None, excludedObjectsList=None):
        for (_, objectCost) in expectedObjectList:
            response = self.client.await_completion(self._runScheduler(workerId))
            objectName = response['result']['name']

            #get the objects with equal costs
            equalCostObjects = [item[0] for item in expectedObjectList
                    if item[1] == objectCost]
            self.assertTrue(objectName in equalCostObjects)

        # This one should be null. That means the 'result' key is not present in the response.
        response = self.client.await_completion(self._runScheduler(workerId))
        self.assertIsNone(response.get('result', None))

        # Add assign to the object. The object should be returned by subsequent 'nextObject' call.
        self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(newAssign))['status'])
        if calculator == 'countassigns':
            newObjectsList = self.getObjectCountsList(newAssign, False, excludedObjectsList)
        else:
            self.assertEqual('OK', self.client.await_completion(self.client.post_compute())['status'])
            newObjectsList = self.getObjectCostsList()

        #get the objects having the maximum priority
        maxPriorityObjects = [item[0] for item in newObjectsList
                    if item[1] == newObjectsList[0][1]]

        response = self.client.await_completion(self._runScheduler(workerId))

        self.assertEqual('OK',response['status'])

        if response.get('result') == None:
            self.assertTrue(len(newAssign) == 0)
        else:
            self.assertTrue(response['result']['name'] in maxPriorityObjects)

    @unittest.skip('Skipping test')
    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddNewAssigns(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'), 
                   ('worker2', 'object1', 'notporn'),
                   ('worker1', 'object2', 'p**n'),
                   ('worker2', 'object2', 'notporn'),
                   ('worker3', 'object2', 'p**n')]

        expectedObjectsList = self.getObjectCountsList(assigns, False)

        newAssigns = [('worker1', 'object3', 'p**n')]
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, newAssigns)

    @unittest.skip('Skipping test')
    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddEmptyAssign(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'),
                   ('worker1', 'object2', 'p**n'),
                   ('worker2', 'object2', 'p**n')]

        expectedObjectsList = self.getObjectCountsList(assigns, False)
        newAssign = []
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, newAssign)

    @unittest.skip('Skipping test')
    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextObject_SameLabelCounts_AddNewAssign(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'),
                   ('worker2', 'object1', 'p**n'),
                   ('worker1', 'object2', 'p**n'),
                   ('worker2', 'object2', 'p**n')]

        expectedObjectsList = self.getObjectCountsList(assigns, False)
        newAssign = [('worker3', 'object0', 'notporn')]
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, newAssign)

    @unittest.skip('Skipping test')
    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextObject_SameLabelCounts_AddEmptyAssign(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'),
                   ('worker2', 'object1', 'p**n'),
                   ('worker1', 'object2', 'p**n'),
                   ('worker2', 'object2', 'p**n')]
        expectedObjectsList = self.getObjectCountsList(assigns, False)
        newAssign = []
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, newAssign)

    @unittest.skip('Skipping test')
    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CostBasedCalculator_GetNextObject_SameObjectCosts(self, algorithm):
        calculator = 'costbased'
        categories = ["cat1", "cat2"]
        categoryPriors = [{"categoryName": "cat1", "value": 0.5}, {"categoryName": "cat2", "value": 0.5}]
        costMatrix =  [{"from": "cat1", "to": "cat2", "value": 1.0}, {"from": "cat1", "to": "cat1", "value": 0.0}, 
                       {"from": "cat2", "to": "cat1", "value": 1.0}, {"from": "cat2", "to": "cat2", "value": 0.0}]
        assigns = [('worker1', 'object1', 'cat1'), 
                   ('worker2', 'object1', 'cat1'),
                   ('worker3', 'object2', 'cat1'),
                   ('worker4', 'object2', 'cat1')]
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors, costMatrix)
        objectCostList = self.getObjectCostsList()
        newAssign = [('worker3', 'object0', 'cat1')]
        self._runTestMethod(calculator, objectCostList, newAssign)

    @unittest.skip('Skipping test')
    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CostBasedCalculator_GetNextObject_DifferentObjectCosts(self, algorithm):
        calculator = 'costbased'
        categories = ["p**n", "notporn"] 
        categoryPriors = [{"categoryName": "p**n", "value": 0.1}, {"categoryName": "notporn", "value": 0.9}]
        costMatrix =  [{"from": "p**n", "to": "notporn", "value": 1.0}, {"from": "p**n", "to": "p**n", "value": 0.0}, 
                       {"from": "notporn", "to": "p**n", "value": 1.0}, {"from": "notporn", "to": "notporn", "value": 0.0}]
        assigns = [('worker0', 'object0', 'notporn'),
                   ('worker1', 'object1', 'p**n'),
                   ('worker2', 'object1', 'p**n')]
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors, costMatrix)
        objectCostList = self.getObjectCostsList()
        newAssign = [('worker3', 'object2', 'p**n')]
        self._runTestMethod(calculator, objectCostList, newAssign)

    @unittest.skip('Skipping test')
    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CostBasedCalculator_GetNextObject_DifferentObjectCosts_AddEmptyLabel(self, algorithm):
        calculator = 'costbased'
        categories = ["p**n", "notporn"]
        categoryPriors = [{"categoryName": "p**n", "value": 0.1}, {"categoryName": "notporn", "value": 0.9}]
        costMatrix =  [{"from": "p**n", "to": "notporn", "value": 1.0}, {"from": "p**n", "to": "p**n", "value": 0.0},
                       {"from": "notporn", "to": "p**n", "value": 1.0}, {"from": "notporn", "to": "notporn", "value": 0.0}]
        assigns = [('worker0', 'object0', 'notporn'),
                   ('worker1', 'object1', 'p**n'),
                   ('worker2', 'object1', 'p**n')]
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors, costMatrix)
        objectCostList = self.getObjectCostsList()

        newAssign = []
        self._runTestMethod(calculator, objectCostList, newAssign)

    @unittest.skip('Skipping test')
    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextWorkerObject_SameLabelCounts(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'),
                   ('worker2', 'object2', 'notporn'),
                   ('worker2', 'object3', 'notporn'),
                   ('worker3', 'object2', 'notporn'),
                   ('worker3', 'object3', 'notporn')
                   ]

        expectedObjectsList = [('object2', 2), ('object3', 2)]
        newAssign = [('worker4', 'object2', 'notporn')]
        excludedObjectsList = ['object1']

        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, newAssign, 'worker1', excludedObjectsList)

    @unittest.skip('Skipping test')
    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextWorkerObject_DifferentLabelCounts(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'),
                   ('worker2', 'object1', 'p**n'),
                   ('worker2', 'object2', 'notporn'),
                   ('worker3', 'object2', 'notporn'),
                   ('worker3', 'object3', 'p**n')]

        expectedObjectsList = [('object3', 1), ('object2', 2)]
        newAssign = [('worker4', 'object2', CATEGORIES[1])]
        excludedObjectsList = ['object1']

        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, newAssign, 'worker1', excludedObjectsList)
class TestNormalScheduler(unittest.TestCase):

    def setUp(self):
        self.client = TroiaClient(ADDRESS)
        self.scheduler = 'normalscheduler'
        self.utils = TestUtils()

    def tearDown(self):
        self.client.delete()

    def _createTestPrereq(self, algorithm, scheduler, calculator, assigns, categories = CATEGORIES, categoryPriors = CATEGORY_PRIORS):
        response = self.client.create(
            categories,
            categoryPriors=categoryPriors,
            algorithm=algorithm,
            scheduler=scheduler,
            prioritycalculator=calculator
        )

        self.assertEqual('OK', response['status'])
        self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(assigns))['status'])
        self.assertEqual('OK', self.client.await_completion(self.client.post_compute())['status'])

    def getObjectCountsList(self, assigns, reverse=False, excludedObjectsList=False):
        #create the dictionary containing the objects and the associated no of assigns
        labelsDict = {}
        for l in assigns:
            if l[1] in labelsDict.keys():
                labelsDict[l[1]] += 1
            else:
                labelsDict[l[1]] = 1

        #sort the objects ascending, based on the no of labels 
        sortedList = sorted(labelsDict.items(), key=itemgetter(1), reverse=reverse)

        if (excludedObjectsList):
            for value in sortedList:
                if value[0] in excludedObjectsList:
                    sortedList.remove(value)
        return sortedList

    def getObjectCostsList(self):
        response = self.client.await_completion(self.client.post_compute())
        self.assertEquals("OK", response['status'])

        response = self.client.await_completion(self.client.get_estimated_objects_cost("ExpectedCost"))
        self.assertEquals("OK", response['status'])

        objectCosts = {}
        for result in response['result']:
            objectCosts[result['objectName']] = result['value']

        #sort the objects descending, based on cost 
        objectCostList = sorted(objectCosts.items(), key=itemgetter(1), reverse=True)
        return objectCostList

    def getAssignedLabels(self):
        response = self.client.await_completion(self.client.get_assigned_labels())
        self.assertEqual('OK', response['status'])
        assignedLabels = [(l['worker'], l['object'], l['label']) for l in response['result']]
        return assignedLabels

    def _runScheduler(self, workerId=None):
        if (workerId):
            return self.client.get_next_worker_object(workerId)
        else:
            return self.client.get_next_object()

    def _runTestMethod(self, calculator, expectedObjectList, newAssign, workerId=None, excludedObjectsList=None):
        for i in xrange(len(expectedObjectList)):
            response = self.client.await_completion(self._runScheduler(workerId))
            self.assertTrue(response['result']['name'] in expectedObjectList)

        # Add assign to the object. The object should be returned by subsequent 'nextObject' call.
        self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(newAssign))['status'])
        response = self.client.await_completion(self._runScheduler(workerId))
        self.assertEqual('OK',response['status'])

        if calculator == 'countassigns':
            newObjectsList = self.getObjectCountsList(self.getAssignedLabels(), False, excludedObjectsList)
        else:
            newObjectsList = self.getObjectCostsList()

        #get the objects having the maximum priority
        maxPriorityObjects = [item[0] for item in newObjectsList
                    if item[1] == newObjectsList[0][1]]

        response = self.client.await_completion(self._runScheduler(workerId))
        self.assertEqual('OK',response['status'])
        self.assertTrue(response['result']['name'] in maxPriorityObjects)

    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddEmptyAssigns(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'),
                   ('worker2', 'object1', 'notporn'),
                   ('worker2', 'object2', 'notporn')]
        sortedList = self.getObjectCountsList(assigns, False)
        minValue = sortedList[0][1]
        expectedObjectsList = [o[0] for o in sortedList if o[1] == minValue]

        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, [])

    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddNewAssigns(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'),
                   ('worker2', 'object1', 'p**n'),
                   ('worker1', 'object2', 'p**n'),
                   ('worker2', 'object2', 'notporn'),
                   ('worker3', 'object2', 'p**n'),
                   ]

        sortedList = self.getObjectCountsList(assigns, False)
        minValue = sortedList[0][1]
        expectedObjectsList = [o[0] for o in sortedList if o[1] == minValue]

        newAssign = [('worker1', 'object3', 'p**n')]
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, newAssign)

    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextObject_SameLabelCounts(self, algorithm):
        calculator = 'countassigns'
        categories = ["cat1", "cat2"] 
        categoryPriors = [{"categoryName": "cat1", "value": 0.5}, {"categoryName": "cat2", "value": 0.5}]
        assigns = [('worker1', 'object1', 'cat1'),
                   ('worker1', 'object2', 'cat1'),
                   ('worker2', 'object1', 'cat2'),
                   ('worker2', 'object2', 'cat2')]
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors)

        sortedList = self.getObjectCountsList(assigns, False)
        minValue = sortedList[0][1]
        expectedObjectList = [o[0] for o in sortedList if o[1] == minValue]
        newAssign = [('worker3', 'object3', 'cat1')]
        self._runTestMethod(calculator, expectedObjectList, newAssign)

    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CostBasedCalculator_GetNextObject_SameCosts(self, algorithm):
        calculator = 'costbased'
        categories = ["cat1", "cat2"]
        categoryPriors = [{"categoryName": "cat1", "value": 0.5}, {"categoryName": "cat2", "value": 0.5}]
        assigns = [('worker1', 'object1', 'cat1'), 
                   ('worker2', 'object1', 'cat1'),
                   ('worker3', 'object2', 'cat1'),
                   ('worker4', 'object2', 'cat1')]
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors)
        sortedList = self.getObjectCostsList()

        minValue = sortedList[0][1]
        expectedObjectList = [o[0] for o in sortedList if o[1] == minValue]

        newAssign = [('worker5', 'object3', 'cat1')]
        self._runTestMethod(calculator, expectedObjectList, newAssign)

    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CostBasedCalculator_GetNextObject_DifferentCosts(self, algorithm):
        calculator = 'costbased'
        categories = ["cat1", "cat2", "cat3"]
        categoryPriors = [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.3},  {"categoryName": "cat3", "value": 0.6}]
        assigns = [('worker1', 'object1', 'cat1'),
                   ('worker2', 'object1', 'cat2'),
                   ('worker2', 'object2', 'cat1'),
                   ('worker3', 'object1', 'cat1'),
                   ('worker3', 'object2', 'cat2')]
        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors)
        sortedList = self.getObjectCostsList()
        minValue = sortedList[0][1]
        expectedObjectList = [o[0] for o in sortedList if o[1] == minValue]
        newAssign = [('worker4', 'object0', 'cat3')]
        self._runTestMethod(calculator, expectedObjectList, newAssign)

    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextWorkerObject_DifferentLabelCounts(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'),
                   ('worker2', 'object1', 'p**n'),
                   ('worker2', 'object2', 'notporn'),
                   ('worker3', 'object2', 'notporn'),
                   ('worker4', 'object2', 'notporn'),
                   ('worker3', 'object3', 'p**n')
                   ]

        expectedObjectsList = ['object3']
        newAssigns = [('worker4', 'object3', CATEGORIES[0])]

        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, newAssigns, 'worker1')

    @data('BDS', 'IDS', 'BMV', 'IMV')
    def test_CountAssignsCalculator_GetNextWorkerObject_SameLabelCounts(self, algorithm):
        calculator = 'countassigns'
        assigns = [('worker1', 'object1', 'p**n'),
                   ('worker2', 'object1', 'p**n'),
                   ('worker2', 'object2', 'notporn'),
                   ('worker3', 'object2', 'p**n'),
                   ('worker3', 'object3', 'notporn'),
                   ('worker4', 'object3', 'p**n')
                   ]

        expectedObjectsList = ['object3', 'object2']
        newAssigns = [('worker4', 'object4', CATEGORIES[0])]

        self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
        self._runTestMethod(calculator, expectedObjectsList, newAssigns, 'worker1')
Example #10
0
class TestJobs(unittest.TestCase):

        def setUp(self):
            self.client = TroiaClient(ADDRESS)

        def tearDown(self):
            self.client.delete()

        def assertJobData(self, response, expAlgorithm, expNoAssigns, expNoGoldObjects, expNoObjects, expNoWorkers):
            self.assertEqual('OK', response['status'])
            self.assertEqual(expAlgorithm, response['result']['initializationData']['algorithm'])
            self.assertEqual(expNoAssigns, response['result']['assigns'])
            self.assertEqual(expNoGoldObjects, response['result']['goldObjects'])
            self.assertEqual(expNoObjects, response['result']['objects'])
            self.assertEqual(expNoWorkers, response['result']['workers'])

        def test_createJob_NoJobType(self):
            response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS)
            response = self.client.await_completion(self.client.get_job_status())
            self.assertJobData(response, 'BDS', 0, 0, 0, 0)

        @data('BDS', 'bdS', 'IDS', 'iDS', 'BMV', 'ImV')
        def test_createJob(self, algorithm):
            response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, algorithm=algorithm)
            self.assertEqual('OK', response['status'])
            self.assertTrue('New job created with ID: RANDOM_' in response['result'])

            response = self.client.await_completion(self.client.get_job_status())
            self.assertJobData(response, algorithm, 0, 0, 0, 0)

        @data(' bDS', 'BdS ', '  BDS  ')
        def test_createJob_TrimParams(self, algorithm):
            response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Unknown algorithm type: [%s]. Did you mean: [bds]' %(algorithm), response['result'])

        def test_createJob_WrongJobType(self):
            response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, algorithm='test')
            self.assertEqual('ERROR', response['status'])
            self.assertTrue('Unknown algorithm type' in response['result'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_NoCategories(self, algorithm):
            response = self.client.create(None, algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('There is no categories collection', response['result'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_EmptyCategories(self, algorithm):
            response = self.client.create([], algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('There should be at least two categories', response['result'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_2Categories_SameCategoryNames(self, algorithm):
            categories = [u'category1', u'category1']
            response = self.client.create(categories, algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Category names should be different', response['result'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_3Categries_SameCategoryNames(self, algorithm):
            categories = [u'category1', u'category1', u'category2']
            response = self.client.create(categories, algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Category names should be different', response['result'])

        def get_priors(self, categories, priors):
            return [{'categoryName': c, "value": p} for c, p in zip(categories, priors)]

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_SumOfPriorsLessThanOne_NoCostMatrix(self, algorithm):
            categories = ["p**n", "notporn"]
            priors = [0.3, 0.5]
            response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Priors should sum up to 1. or not to be given (therefore we initialize the priors to be uniform across classes)', response['result'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_SumOfPriorsGreaterThanOne_NoCostMatrix(self, algorithm):
            categories = ["p**n", "notporn"]
            priors = [0.6, 0.5]
            response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Priors should sum up to 1. or not to be given (therefore we initialize the priors to be uniform across classes)', response['result'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_SumOfPriorsEqualsOne_NoCostMatrix(self, algorithm):
            categories = ["p**n", "notporn"]
            priors = [0.234, 0.766]
            response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), algorithm=algorithm)
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_job_status())
            self.assertJobData(response, algorithm, 0, 0, 0, 0)

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_SumOfPriorsLessThanOne_WithCostMatrix(self, algorithm):
            categories = ["p**n", "notporn"]
            priors = [0.3, 0.5]
            response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), costMatrix=COST_MATRIX, algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Priors should sum up to 1. or not to be given (therefore we initialize the priors to be uniform across classes)', response['result'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_SumOfPriorsGreaterThanOne_WithCostMatrix(self, algorithm):
            categories = ["p**n", "notporn"]
            priors = [0.6, 0.5]
            response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), costMatrix=COST_MATRIX, algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Priors should sum up to 1. or not to be given (therefore we initialize the priors to be uniform across classes)', response['result'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_SumOfPriorsEqualsOne_WithCostMatrix(self, algorithm):
            categories = ["p**n", "notporn"]
            priors = [0.1234, 0.8766]
            response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), costMatrix=COST_MATRIX, algorithm=algorithm)
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_job_status())
            self.assertJobData(response, algorithm, 0, 0, 0, 0)

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_NoPriors_NoCostMatrix(self, algorithm):
            categories = ["p**n", "notporn"]

            response = self.client.create(categories, algorithm=algorithm)
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_job_status())
            self.assertJobData(response, algorithm, 0, 0, 0, 0)

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_NoPriors_WithCostMatrix(self, algorithm):
            categories = ['p**n', 'notporn']
            response = self.client.create(categories, costMatrix=COST_MATRIX, algorithm=algorithm)
            self.assertEqual('OK', response['status'])

            response = self.client.await_completion(self.client.get_job_status())
            self.assertJobData(response, algorithm, 0, 0, 0, 0)

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_createJob_CostMatrixContainsNotExistingCategories(self, algorithm):
            categories = [u'category1', u'category2']
            response = self.client.create(categories, costMatrix=COST_MATRIX, algorithm=algorithm)
            self.assertEqual('ERROR', response['status'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_deleteJob_ExistingJobId(self, algorithm):
            response = self.client.create(CATEGORIES, algorithm=algorithm)
            self.assertEqual('OK', response['status'])
            self.assertEqual('New job created with ID: ' + self.client.jid, response['result'])

            response = self.client.delete()
            self.assertEqual('OK', response['status'])
            self.assertEqual('Removed job with ID: ' + self.client.jid, response['result'])

        @data('BDS', 'IDS', 'BMV', 'IMV')
        def test_deleteJob_NonExistingJobId(self, algorithm):
            response = self.client.create(CATEGORIES, algorithm=algorithm)
            self.assertEqual('OK', response['status'])
            self.assertEqual('New job created with ID: ' + self.client.jid, response['result'])

            response = self.client.delete('NotExistingJob')
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Job with ID NotExistingJob does not exist', response['result'])
class TestAssignedLabels(unittest.TestCase):
        def setUp(self):
            self.client = TroiaClient(ADDRESS)

        def tearDown(self):
            self.client.delete()

        def _test_method(self, assigned_labels):
            #post the assigned labels
            response = self.client.await_completion(self.client.post_assigned_labels(assigned_labels))
            self.assertEqual('OK', response['status'])

            #get the assigned labels
            response = self.client.await_completion(self.client.get_assigned_labels())
            self.assertEqual('OK', response['status'])
            self.assertEqual(len(response['result']), len(assigned_labels))
            results = [(l['worker'], l['object'], l['label']) for l in response['result']]
            for label in assigned_labels:
                self.assertTrue(label in results)

        def test_AddGetEmptyAssignedLabels(self):
            response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
            self.assertEqual('OK', response['status'])
            self._test_method([])

        def test_AddAssignedLabelsWithInvalidCategory(self):
            categories = ["category1", "category2"]
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])

            #post the assigned labels
            assignedLabels = [('worker1', 'url1', 'category3')]
            response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
            self.assertEqual('ERROR', response['status'])

            #get the assigned labels
            response = self.client.await_completion(self.client.get_assigned_labels())
            self.assertEqual('OK', response['status'])
            result = response['result']
            self.assertFalse(result)

        def test_AddGetAssignedLabels_LongWorkerNames(self):
            categories = ['category1', "category2"]
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])
            response = self.client.await_completion(self.client.post_assigned_labels([('hjkdhfhdfgjkshfghdsgffgfhfghgjhghjgjgjgjgjgjgjldkgjhjkdhfhdfgjkshfghdsgffgfhfghgjhghjgjgjgjgjgjgjldkg', 'object_dsjgfhgfhgfhhjdfhgkgkhjkfklsdjfkljsdfj', 'category1')]))
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Internal error: Worker name should be shorter than 100 chars', response['result'])

        def test_AddGetAssignedLabels_LongObjectNames(self):
            categories = ['category1', "category2"]
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])
            response = self.client.await_completion(self.client.post_assigned_labels([('hjkdhfhdfgjkshfghdsgffgfhfghgjhghjgkgj', 'object_dsjgfhgfhgfhhjdfhgjghghkgkhjkfklsdjfkljsdfjobjectdsjgfhgfhgfhhjdfhgjghghkgkhjkfklsdjfkljsdfjwd', 'category1')]))
            self.assertEqual('ERROR', response['status'])
            self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result'])


        def test_AddGetAssignedLabels_PrintableASCII_RegularChars(self):
            response = self.client.create(CATEGORIES)
            self.assertEqual('OK', response['status'])
            self._test_method(ASSIGNED_LABELS)

        def test_AddGetAssignedLabels_PrintableASCII_SpecialChars(self):
            categories = ['category1', "category2", "category3"]
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])

            self._test_method([('a%a', '~!@#%^&*()_+=-[]{}:<>,./', 'category1'),
                ('b%%b', '%%%', 'category2'),
                ('c%%!<>c', '~!@#^&*[](){}-_+=<>?/.,:', 'category3')])

        def test_AddGetAssignedLabels_ExtendedASCIIChars(self):
            categories = ['category1', "category2", "category3"]
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])

            self._test_method([(u'®¶', u'œŒ', 'category1'),
                (u'ÀÆË', u'™ž¤©', 'category2'),
                (u'ëñ', u'µ¼Úæ', 'category3')])

        def test_AddGetAssignedLabels_UnicodeChars(self):
            categories = ['category1', "category2", "category3"]
            response = self.client.create(categories)
            self.assertEqual('OK', response['status'])
            self._test_method([(u'ૉେஇ', u'ΨҖӖմ؂څ', 'category1'),
                (u'ూഹ', u'ܬआਖ਼', 'category2')])
class TestQualitySensitivePayments(unittest.TestCase):
    def setUp(self):
        self.client = TroiaClient(ADDRESS)

    def tearDown(self):
        self.client.delete()

    def testQSP_ZeroWorkerQUality(self):
        assignedLabels = [
                            ('worker1', 'url1', 'cat2'),
                            ('worker1', 'url2', 'cat1'),
                            ('worker2', 'url1', 'cat1'),
                            ('worker2', 'url2', 'cat1'),
                            ('worker3', 'url1', 'cat1'),
                            ('worker3', 'url2', 'cat2')]
        goldLabels = [
                      ('url1', 'cat1'),
                      ('url2', 'cat2')
                      ]
        response = self.client.create(["cat1", "cat2"])
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_gold_data(goldLabels))
        self.assertEqual('OK', response['status'])

        response = self.client.post_compute()
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
        self.assertEqual('OK', response['status'])
        pprint(response)

        response = self.client.await_completion(self.client.get_workers_quality_payment())
        pprint(response)

    def testQSP_NegativeWorkerQuality(self):
        assignedLabels = [
                            ('worker1', 'url1', 'cat2'),
                            ('worker1', 'url2', 'cat1'),
                            ('worker1', 'url3', 'cat2'),
                            ('worker2', 'url1', 'cat1'),
                            ('worker2', 'url2', 'cat1'),
                            ('worker2', 'url3', 'cat1'),
                            ('worker3', 'url1', 'cat1'),
                            ('worker3', 'url2', 'cat2'),
                            ('worker3', 'url3', 'cat1')
                            ]
        goldLabels = [
                      ('url1', 'cat1'),
                      ('url2', 'cat2'),
                      ('url3', 'cat1')
                      ]
        response = self.client.create(["cat1", "cat2"], categoryPriors= [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.9}])
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.post_gold_data(goldLabels))
        self.assertEqual('OK', response['status'])

        response = self.client.post_compute()
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.get_workers_confusion_matrix())
        pprint(response)

        response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
        self.assertEqual('OK', response['status'])
        pprint(response)

        response = self.client.await_completion(self.client.get_workers_quality_payment())
        pprint(response)
from client.gal import TroiaClient
from nominalJobsTests.testSettings import *


def check_status(res):
    if res['status'] != "OK" or res['result']['job_storage_status'] != "OK":
        print "FAILURE"

if __name__ == "__main__":
    client = TroiaClient(ADDRESS)
    check_status(client.status())
    client.create(CATEGORIES)
    client.await_completion(client.post_assigned_labels(ASSIGNED_LABELS))
    client.await_completion(client.post_gold_data(GOLD_SAMPLES))
    client.await_completion(client.post_compute())
#    client.await_completion(client.get_predictions_objects())
#    client.await_completion(client.get_prediction_workers_quality())
    client.await_completion(client.get_prediction_zip())
    client.delete()
    check_status(client.status())
class BugRegressionTests(unittest.TestCase):

    def setUp(self):
        self.client = TroiaClient(ADDRESS)

    def tearDown(self):
        self.client.delete()

    def test_GH_83(self):
        """ 
            Test for github issue https://github.com/ipeirotis/Troia-Server/issues/83
            Thanks to PeccatorImpius for reporting
        """
        categories = ['yes', 'no', 'blank']
        response = self.client.create(categories,
                                      prioritycalculator="CostBased",
                                      costMatrix=[{"value":5.0, "to":"blank", "from":"blank"},
                                                   {"value":5.0, "to":"no", "from":"blank"},
                                                   {"value":5.0, "to":"yes", "from":"blank"},
                                                   {"value":1.0, "to":"blank", "from":"no"},
                                                   {"value":0.0, "to":"no", "from":"no"},
                                                   {"value":1.0, "to":"yes", "from":"no"},
                                                   {"value":1.0, "to":"blank", "from":"yes"},
                                                   {"value":1.0, "to":"no", "from":"yes"},
                                                   {"value":0.0, "to":"yes", "from":"yes"}],
                                      epsilon=1.0E-4,
                                      iterations=10,
                                      algorithm='IDS',
                                      scheduler='NormalScheduler')
        self.assertEqual('OK', response['status'])
        assignedLabels = [
            ('A2G061ZJVQXGE', '21K74J79KQW6KZ5ZI8OFKD455U4HJ6', 'blank'),
            ('A2G061ZJVQXGE', '2FWPLSP75KLMPN1Y0O8QMQVJZMCZTJ', 'yes'),
            ('A224TK7J4KA1LV', '283PHIT3HVWAEB01E1AQA1727RZCKR', 'yes'),
            ('A224TK7J4KA1LV', '2S3YHVI44OS23CC3P15Z0BAPDTU4Y4', 'yes'),
            ('A224TK7J4KA1LV', '2NCXFCD45XCECEVNZ7BIV3TGXMSYWJ', 'yes'),
            ('A224TK7J4KA1LV', '295QQ7Q670519BUI3FTTYC1XUY7WC6', 'no'),
            ('A224TK7J4KA1LV', '255NVRH1KHO9X796JKEVHLKWFFDZQ5', 'yes'),
            ('A224TK7J4KA1LV', '21Z1Q6SVQ8H8RD8PIZWOJVCI99NN4U', 'yes'),
            ('A224TK7J4KA1LV', '2JGU98JHSTLBMCIANX0IWBJER94EQE', 'yes'),
            ('A224TK7J4KA1LV', '2Z3KH1Q6SVQ80ZQH28XLDOBVK0S2LC', 'yes'),
            ('A224TK7J4KA1LV', '2P3Z6R70G5R9R4QEF9574SSTVG2P9O', 'yes'),
            ('A224TK7J4KA1LV', '25UCF0CP5KXPC95DUBPGCPM7VGWWX9', 'yes'),
            ('A224TK7J4KA1LV', '2AOYTWX4H3H282M8LN7IEIJRJNW4ZT', 'no'),
            ('A224TK7J4KA1LV', '2J5JQ2172Z9998O6WZ3VMBU1B1FZR6', 'no'),
            ('A224TK7J4KA1LV', '2SIHFL42J1LYOZT2ELD0QGFKPVS2NE', 'yes'),
            ('A224TK7J4KA1LV', '2W36VCPWZ9RNWN0J7FJ70N3D1YKPA3', 'yes'),
            ('A224TK7J4KA1LV', '2F8S1CSTMOFXA4AZUQSE6DNXRXCROW', 'yes'),
            ('A224TK7J4KA1LV', '2TRKSSZVXX2QNWCPW4FYONIW95IN5Z', 'no'),
            ('A224TK7J4KA1LV', '2L13NAB6BFMFYFGTU7STRFDGMA2HU2', 'yes'),
            ('A224TK7J4KA1LV', '2B3RGNTBJ3MMWONA1LE6AKHBH439VT', 'yes'),
            ('A224TK7J4KA1LV', '24ZMVHVKCJ01KBK2GXY9SQW69QD758', 'yes'),
            ('A224TK7J4KA1LV', '2V99Y8TNKIMZBDNFRVJ2WM0POD59W3', 'yes'),
            ('A224TK7J4KA1LV', '2FK4E6AUBXHWB8PAXN0VJZWDZ3NX1B', 'yes'),
            ('A224TK7J4KA1LV', '2WAVGULF395SFX5BDTJ3VYNJJG5PBY', 'yes'),
            ('A224TK7J4KA1LV', '2DMZMAZHHRRLYHIJZ1MRP1KHWR4CLU', 'yes'),
            ('A224TK7J4KA1LV', '27E618N46UXFV4M09Q5TVDSN128RPD', 'yes'),
            ('A224TK7J4KA1LV', '2J9CJK63ZVE30IIF6W1QHJL5ZL59Y2', 'yes'),
            ('A224TK7J4KA1LV', '2DS6X3YOCCF0VGNF93KIVIIX6YURQ4', 'yes'),
            ('A224TK7J4KA1LV', '23ZGYQ4J3NABP2XHRTFY6IT1115PC5', 'yes'),
            ('A224TK7J4KA1LV', '2PFMB3Q39Z0BPLBJI7VE3STVZ26UJK', 'yes'),
            ('A224TK7J4KA1LV', '283FQ0ONNVRHKBZJLS7RJ76N3RBULM', 'no'),
            ('A224TK7J4KA1LV', '2F8S1CSTMOFXA4AZUQSE6DNXRXFORW', 'yes'),
            ('A224TK7J4KA1LV', '21UJ011K274JQ02L8KS8V46U5X5BD9', 'no'),
            ('A224TK7J4KA1LV', '2QAKMT6YTWX40UZX1PVDH9GIE0C0VB', 'yes'),
            ('A224TK7J4KA1LV', '2C18EBDPGFL6E37RBNINLWIO3JLH2M', 'yes'),
            ('A224TK7J4KA1LV', '2Y6VR14IXKO240KCQQNAEX3YWU59AV', 'yes'),
            ('A224TK7J4KA1LV', '2P7JWKUDD8XMUU8YLDRBEUTOEBXMB2', 'yes'),
            ('A224TK7J4KA1LV', '2525UFJ51Y7QXFHAA4J1KSTMWXQ9CX', 'yes'),
            ('A224TK7J4KA1LV', '2NCXFCD45XCECEVNZ7BIV3TGXMQWYF', 'yes'),
            ('A224TK7J4KA1LV', '27ZM8JIA0RYN20KIVYD0DBMJ1C8507', 'yes'),
            ('A224TK7J4KA1LV', '2I63W5XH0JPPB9KFASGL0P75S3FEKP', 'yes'),
            ('A224TK7J4KA1LV', '21RRJ85OZIZE0DS510FOY5EWXFB9XW', 'yes'),
            ('A224TK7J4KA1LV', '2LAMEZZ2MIKM60C31NP3GTWW50TN6F', 'yes'),
            ('A224TK7J4KA1LV', '241KM05BMJTU0PEXS7G9ZA7SDEFEJQ', 'yes'),
            ('A224TK7J4KA1LV', '2UZDM25L8I9NINOCQDXOM4QWFBYHRY', 'yes'),
            ('A224TK7J4KA1LV', '2J1E1M7PKK9LANOJBO30CLZXX3YZSU', 'yes'),
            ('A224TK7J4KA1LV', '2LWGDHDM25L8105U8K8E76OEC8MEO6', 'no'),
            ('A224TK7J4KA1LV', '2XANTL0XULRG6KTEF0DD55FPF5X2OT', 'yes'),
            ('A224TK7J4KA1LV', '2FC98JHSTLB34RX6VN9OJJEJZWORFD', 'yes'),
            ('A224TK7J4KA1LV', '2D2UDD8XMB3QM0HVNKLTW6T4MDIEPN', 'yes'),
            ('A224TK7J4KA1LV', '2C73TVOK5UFJOSG22SFZNYQS9UI743', 'yes'),
            ('A224TK7J4KA1LV', '25XXRDS4IC1EH45SVTD19A2I3GUWZ6', 'no'),
            ('A224TK7J4KA1LV', '2PG1THV8ER9YRK5FU0QSU5KFPAT2PA', 'yes'),
            ('A224TK7J4KA1LV', '2C79746OQ1SN9HPLILR59QKCV9RN39', 'yes'),
            ('A224TK7J4KA1LV', '2YDKCJ011K27NAP4W4N698N4ECQ9BQ', 'yes'),
            ('A224TK7J4KA1LV', '2XGQ4J3NAB6BYDXA0CPI11TJNV6ER,F', 'yes'),
            ('A224TK7J4KA1LV', '23K5L8I9NZW605H10SVQ47T8ER6UKJ', 'yes'),
            ('A224TK7J4KA1LV', '2ZCUU00OQKKAN90NK6TUA680MEQWWG', 'yes'),
            ('A224TK7J4KA1LV', '2P1HSTLB3L0FUARD0PAERREVMNHUI8', 'yes'),
            ('A224TK7J4KA1LV', '2581SNQQ7Q67JWJLWQER92TQKJN9TD', 'yes'),
            ('A224TK7J4KA1LV', '2LISPW1INMHGHHMEF11BEBFMNXHH4I', 'yes'),
            ('A224TK7J4KA1LV', '2SIHFL42J1LYOZT2ELD0QGFKPVRN2Y', 'yes'),
            ('A224TK7J4KA1LV', '2ZRNZW6HEZ6OXV8RJ7Z6HGUMAJRZPE', 'yes'),
            ('A224TK7J4KA1LV', '214T6YTWX4H30T76GR09OI6IR91X2S', 'yes'),
            ('A224TK7J4KA1LV', '2T5HMVHVKCJ0KS2XJIA7HKQWEJ1646', 'yes'),
            ('A224TK7J4KA1LV', '2ALDHJHP4BDD37MTK5JXHX3GBNY4XW', 'yes'),
            ('A224TK7J4KA1LV', '2IDP9746OQ1S6H822KY0D1QKK5I2ME', 'yes'),
            ('A224TK7J4KA1LV', '2WIOHOVR14IX3FKGLG8EKWA65LO76O', 'yes'),
            ('A224TK7J4KA1LV', '2GYKPEIB9BAW4J4S14WV5UD3UBHX08', 'yes'),
            ('A224TK7J4KA1LV', '2WQ06UFBNFSVZL3AFNWS46NG9XLH3X', 'yes'),
            ('A224TK7J4KA1LV', '2L13NAB6BFMFYFGTU7STRFDGMA2UHF', 'yes'),
            ('A224TK7J4KA1LV', '2ALDHJHP4BDD37MTK5JXHX3GBN04XY', 'no'),
            ('A224TK7J4KA1LV', '2F4NCWYB49F9BJAC9FSJWNSTDZYOSK', 'yes'),
            ('A224TK7J4KA1LV', '2BXBNFSVGULFM0NN8KEG9FS3VGE7LX', 'yes'),
            ('A224TK7J4KA1LV', '20JLY58B727MJ9YAWV41EYFS3XXU9S', 'no'),
            ('A224TK7J4KA1LV', '2FKAOO2GMMEG7FYL4QD6QAA20W877N', 'yes'),
            ('A224TK7J4KA1LV', '20JLY58B727MJ9YAWV41EYFS3XU9U4', 'yes'),
            ('A224TK7J4KA1LV', '2A79RA7S5WM1P0GN0TY0RMHLI0JZUA', 'yes'),
            ('A224TK7J4KA1LV', '2AOYTWX4H3H282M8LN7IEIJRJNY4ZV', 'yes'),
            ('A2ZUENR4ZLC3MN', '2K5AB6BFMFFOHP0OD7AFLGESKIHJW8', 'yes')]

        response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
        self.assertEqual('OK', response['status'])

        response = self.client.await_completion(self.client.get_estimated_workers_quality())
        self.assertEqual('OK', response['status'])

    def test_GH_88(self):
        """
            Test for github issue https://github.com/ipeirotis/Troia-Server/issues/88
            Thanks to PeccatorImpius for reporting
        """
        categories = ['blank', 'confirm_closed', 'confirm_no', 'confirm_nonrestaurant', 'confirm_yes']
        algorithm = "BDS" # TODO XXX FIXME currently it won't work on IDS
        response = self.client.create(categories,
                                      prioritycalculator="CostBased",
                                      costMatrix=[ {"value":5.0,  "to":"blank", "from":"blank"},
                                                   {"value":5.0, "to":"confirm_closed", "from":"blank"},
                                                   {"value":5.0, "to":"confirm_no", "from":"blank"},
                                                   {"value":5.0, "to":"confirm_nonrestaurant", "from":"blank"},
                                                   {"value":5.0, "to":"confirm_yes", "from":"blank"},
                                                   {"value":5.0, "to":"blank", "from":"confirm_closed"},
                                                   {"value":0.0, "to":"confirm_closed", "from":"confirm_closed"},
                                                   {"value":5.0, "to":"confirm_no", "from":"confirm_closed"},
                                                   {"value":5.0, "to":"confirm_nonrestaurant", "from":"confirm_closed"},
                                                   {"value":5.0, "to":"confirm_yes", "from":"confirm_closed"},
                                                   {"value":1.0, "to":"blank", "from":"confirm_no"},
                                                   {"value":1.0, "to":"confirm_closed", "from":"confirm_no"},
                                                   {"value":0.0, "to":"confirm_no", "from":"confirm_no"},
                                                   {"value":1.0, "to":"confirm_nonrestaurant", "from":"confirm_no"},
                                                   {"value":1.0, "to":"confirm_yes", "from":"confirm_no"},
                                                   {"value":5.0, "to":"blank", "from":"confirm_nonrestaurant"},
                                                   {"value":5.0, "to":"confirm_closed", "from":"confirm_nonrestaurant"},
                                                   {"value":5.0, "to":"confirm_no", "from":"confirm_nonrestaurant"},
                                                   {"value":0.0, "to":"confirm_nonrestaurant", "from":"confirm_nonrestaurant"},
                                                   {"value":5.0, "to":"confirm_yes", "from":"confirm_nonrestaurant"},
                                                   {"value":1.0, "to":"blank", "from":"confirm_yes"},
                                                   {"value":1.0, "to":"confirm_closed", "from":"confirm_yes"},
                                                   {"value":1.0, "to":"confirm_no", "from":"confirm_yes"},
                                                   {"value":1.0, "to":"confirm_nonrestaurant", "from":"confirm_yes"},
                                                   {"value":0.0, "to":"confirm_yes", "from":"confirm_yes"}],
                                      epsilon=0.0001,
                                      iterations=10,
                                      algorithm=algorithm,
                                      scheduler='NormalScheduler')
        self.assertEqual('OK', response['status'])

        assignedLabels = [
                          ('A1R7CJMWXC79UO', '2OVZKPFE4EGD044XHZZIHNZWGWV7HC', 'confirm_yes'),
                          ('A2VRQML8Q3XYP4', '2OVZKPFE4EGD044XHZZIHNZWGWV7HC', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2OVZKPFE4EGD044XHZZIHNZWGWV7HC', 'confirm_yes'),
                          ('A3TXO6RKFIDFUV', '2Z3KH1Q6SVQ80ZQH28XLDOBVMXHL2G', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2Z3KH1Q6SVQ80ZQH28XLDOBVMXHL2G', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2Z3KH1Q6SVQ80ZQH28XLDOBVMXHL2G', 'confirm_yes'),
                          ('A1FQYUBCBNTQX7', '2V6ZFYQS1CST5FXS3RJ4QC1E8S3KN3', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2V6ZFYQS1CST5FXS3RJ4QC1E8S3KN3', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2V6ZFYQS1CST5FXS3RJ4QC1E8S3KN3', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2MCFJ51Y7QEOI6GL4F3S1MOF76TBEL', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2MCFJ51Y7QEOI6GL4F3S1MOF76TBEL', 'confirm_yes'),
                          ('A3TXO6RKFIDFUV', '2MCFJ51Y7QEOI6GL4F3S1MOF76TBEL', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2S4JTUHYW2GT8095J6WWU169874PK4', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2S4JTUHYW2GT8095J6WWU169874PK4', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2S4JTUHYW2GT8095J6WWU169874PK4', 'confirm_no'),
                          ('A3PRF4IIM2IQN2', '2S3YHVI44OS23CC3P15Z0BAPFQI4YO', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2S3YHVI44OS23CC3P15Z0BAPFQI4YO', 'confirm_closed'),
                          ('AH5ZIMHL7TNWC', '2S3YHVI44OS23CC3P15Z0BAPFQI4YO', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2KXDEY5COW0LZIHYKQIFTC6VX9G4VF', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2KXDEY5COW0LZIHYKQIFTC6VX9G4VF', 'confirm_yes'),
                          ('A2KET1HL1COET5', '2KXDEY5COW0LZIHYKQIFTC6VX9G4VF', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2FT49F9SSSHXKS1JZ6K5P5F8TJ4TX2', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2FT49F9SSSHXKS1JZ6K5P5F8TJ4TX2', 'confirm_no'),
                          ('A3TXO6RKFIDFUV', '2FT49F9SSSHXKS1JZ6K5P5F8TJ4TX2', 'confirm_no'),
                          ('A3PRF4IIM2IQN2', '27G2RCJK63ZVXUZMCYLIIQ9JVK87WC', 'confirm_yes'),
                          ('A19BL9GZGXFWFT', '27G2RCJK63ZVXUZMCYLIIQ9JVK87WC', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '27G2RCJK63ZVXUZMCYLIIQ9JVK87WC', 'confirm_no'),
                          ('AH5ZIMHL7TNWC', '2JODG67D1X3VJ62VO2B2RE1MH40HAC', 'confirm_no'),
                          ('A31X3JCHS0BPFJ', '2JODG67D1X3VJ62VO2B2RE1MH40HAC', 'confirm_yes'),
                          ('A2VRQML8Q3XYP4', '2JODG67D1X3VJ62VO2B2RE1MH40HAC', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '26ZIT3HVWAVK1XKIV4T1F2Z9J5XEMD', 'confirm_closed'),
                          ('A3TXO6RKFIDFUV', '26ZIT3HVWAVK1XKIV4T1F2Z9J5XEMD', 'confirm_closed'),
                          ('AH5ZIMHL7TNWC', '26ZIT3HVWAVK1XKIV4T1F2Z9J5XEMD', 'confirm_nonrestaurant'),
                          ('A3PRF4IIM2IQN2', '20N1Y7QEOZFY9JJ747DONXRD2JYEH4', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '20N1Y7QEOZFY9JJ747DONXRD2JYEH4', 'confirm_yes'),
                          ('A2KET1HL1COET5', '20N1Y7QEOZFY9JJ747DONXRD2JYEH4', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '241KM05BMJTU0PEXS7G9ZA7SFB2EJ9', 'confirm_yes'),
                          ('A2L2ZXRHK3SNOP', '241KM05BMJTU0PEXS7G9ZA7SFB2EJ9', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '241KM05BMJTU0PEXS7G9ZA7SFB2EJ9', 'confirm_no'),
                          ('A3PRF4IIM2IQN2', '2YDWAVKI62NJ9TJ2ED09YH6BU0SRJZ', 'confirm_yes'),
                          ('A19BL9GZGXFWFT', '2YDWAVKI62NJ9TJ2ED09YH6BU0SRJZ', 'confirm_closed'),
                          ('AH5ZIMHL7TNWC', '2YDWAVKI62NJ9TJ2ED09YH6BU0SRJZ', 'confirm_no'),
                          ('A3PRF4IIM2IQN2', '2JV21O3W5XH02G7NUGBYMPLSZMLHB7', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2JV21O3W5XH02G7NUGBYMPLSZMLHB7', 'confirm_yes'),
                          ('A2L2ZXRHK3SNOP', '2JV21O3W5XH02G7NUGBYMPLSZMLHB7', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2JS1QP6AUC26W7O2PFO330FKAR5811', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2JS1QP6AUC26W7O2PFO330FKAR5811', 'confirm_yes'),
                          ('A1I3CXC17NIRWB', '2JS1QP6AUC26W7O2PFO330FKAR5811', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2CIFK0COK2JEKDPKWY0LZW6O9PSRKY', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2CIFK0COK2JEKDPKWY0LZW6O9PSRKY', 'confirm_yes'),
                          ('A2KET1HL1COET5', '2CIFK0COK2JEKDPKWY0LZW6O9PSRKY', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2IJK274J79KQFXJ3ZIXU5FCDEKDHFP', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2IJK274J79KQFXJ3ZIXU5FCDEKDHFP', 'confirm_no'),
                          ('A2L2ZXRHK3SNOP', '2IJK274J79KQFXJ3ZIXU5FCDEKDHFP', 'confirm_yes'),
                          ('A3TXO6RKFIDFUV', '234EGOOGQSCMP9S5E65I2UU0A36EEO', 'confirm_closed'),
                          ('A3PRF4IIM2IQN2', '234EGOOGQSCMP9S5E65I2UU0A36EEO', 'confirm_closed'),
                          ('A31X3JCHS0BPFJ', '234EGOOGQSCMP9S5E65I2UU0A36EEO', 'confirm_closed'),
                          ('A3TXO6RKFIDFUV', '23QJIA0RYNJ9LE1FYEWBUJTURDD27P', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '23QJIA0RYNJ9LE1FYEWBUJTURDD27P', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '23QJIA0RYNJ9LE1FYEWBUJTURDD27P', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2S5VP3TVOK5UYANWALHEWZFY07I52D', 'confirm_no'),
                          ('A2L2ZXRHK3SNOP', '2S5VP3TVOK5UYANWALHEWZFY07I52D', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2S5VP3TVOK5UYANWALHEWZFY07I52D', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2ETAT2D5NQYNO6LLCX751YMNMBF84E', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2ETAT2D5NQYNO6LLCX751YMNMBF84E', 'confirm_no'),
                          ('A2L2ZXRHK3SNOP', '2ETAT2D5NQYNO6LLCX751YMNMBF84E', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2Z2MN9U8P9Y3RKER9WUS2YIMAY9EVU', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2Z2MN9U8P9Y3RKER9WUS2YIMAY9EVU', 'confirm_no'),
                          ('A1I3CXC17NIRWB', '2Z2MN9U8P9Y3RKER9WUS2YIMAY9EVU', 'confirm_yes'),
                          ('ABT7QTMIYXYO0', '2GXYQS1CSTMOYO984I9C9EYDXCZPM9', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2GXYQS1CSTMOYO984I9C9EYDXCZPM9', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2GXYQS1CSTMOYO984I9C9EYDXCZPM9', 'confirm_no'),
                          ('AH5ZIMHL7TNWC', '2BLK4F0OHOVRKV0SW2TLH2HEMBR23K', 'blank'),
                          ('A3PRF4IIM2IQN2', '2BLK4F0OHOVRKV0SW2TLH2HEMBR23K', 'confirm_yes'),
                          ('A2L2ZXRHK3SNOP', '2BLK4F0OHOVRKV0SW2TLH2HEMBR23K', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2X6OGQSCM6IATTA9U8LU80OQUZQHHN', 'confirm_yes'),
                          ('A2KET1HL1COET5', '2X6OGQSCM6IATTA9U8LU80OQUZQHHN', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2X6OGQSCM6IATTA9U8LU80OQUZQHHN', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '29UWY2AOO2GM55YJ0UHSKM6IKPI442', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '29UWY2AOO2GM55YJ0UHSKM6IKPI442', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '29UWY2AOO2GM55YJ0UHSKM6IKPI442', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '2DJVP9746OQ1BE8LJ4X7851QUR3L1Z', 'confirm_yes'),
                          ('ABT7QTMIYXYO0', '2DJVP9746OQ1BE8LJ4X7851QUR3L1Z', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2DJVP9746OQ1BE8LJ4X7851QUR3L1Z', 'confirm_yes'),
                          ('A31X3JCHS0BPFJ', '25XXRDS4IC1EH45SVTD19A2I5DKWZS', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '25XXRDS4IC1EH45SVTD19A2I5DKWZS', 'confirm_no'),
                          ('AH5ZIMHL7TNWC', '25XXRDS4IC1EH45SVTD19A2I5DKWZS', 'confirm_no'),
                          ('A2VRQML8Q3XYP4', '2DABRI8IUB2YD0QET6KLJ3L0PQZH56', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2DABRI8IUB2YD0QET6KLJ3L0PQZH56', 'confirm_no'),
                          ('A3PRF4IIM2IQN2', '2DABRI8IUB2YD0QET6KLJ3L0PQZH56', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2TOSK9RJ85OZ1QWCYO1PUOOQFTCU60', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2TOSK9RJ85OZ1QWCYO1PUOOQFTCU60', 'confirm_yes'),
                          ('A2L2ZXRHK3SNOP', '2TOSK9RJ85OZ1QWCYO1PUOOQFTCU60', 'confirm_yes'),
                          ('A3PRF4IIM2IQN2', '2IBHV8ER9Y8T6B0HB6D5SFHSC334RM', 'confirm_yes'),
                          ('ABT7QTMIYXYO0', '2IBHV8ER9Y8T6B0HB6D5SFHSC334RM', 'confirm_yes'),
                          ('AH5ZIMHL7TNWC', '2IBHV8ER9Y8T6B0HB6D5SFHSC334RM', 'confirm_no'),
                          ('A3TXO6RKFIDFUV', '294EZZ2MIKMNSLQKLCU81WWXSI97O0', 'confirm_yes'),
                          ('A19BL9GZGXFWFT', '294EZZ2MIKMNSLQKLCU81WWXSI97O0', 'confirm_yes'),
                          ('A1I3CXC17NIRWB', '294EZZ2MIKMNSLQKLCU81WWXSI97O0', 'confirm_yes'),
                          ('A2VRQML8Q3XYP4', '2AOYTWX4H3H282M8LN7IEIJRLKM4ZF', 'confirm_yes')]

        response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
        self.assertEqual('OK', response['status'])
        
        if algorithm == "BDS":
            self.client.await_completion(self.client.post_compute())

        response = self.client.await_completion(self.client.get_estimated_workers_quality())
        self.assertEqual('OK', response['status'])
        for w in response['result']:
            val = float(w['value'])
            self.assertFalse(math.isnan(val))