class TestGoldLabels(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) response = self.client.create(CATEGORIES) self.assertEqual("OK", response["status"]) def tearDown(self): self.client.delete() def _test_method(self, goldLabels): response = self.client.await_completion(self.client.post_gold_data(goldLabels)) self.assertEqual("OK", response["status"]) # get the gold labels response = self.client.await_completion(self.client.get_gold_data()) self.assertEqual("OK", response["status"]) result = response["result"] self.assertEqual(len(goldLabels), len(result)) results = [tuple(receivedLabel.values()) for receivedLabel in response["result"]] for label in goldLabels: self.assertTrue(label in results) def test_AddGetEmptyGoldLabels(self): self._test_method([]) def test_AddGetGoldLabel_LongLabelName(self): response = self.client.await_completion( self.client.post_gold_data( [ ( "sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs", "notporn", ) ] ) ) self.assertEqual("ERROR", response["status"]) self.assertEqual("Internal error: Object name should be shorter than 100 chars", response["result"]) def test_AddGetGoldLabel_PrintableASCII_RegularChars(self): self._test_method([("url1", "notporn")]) def test_AddGetGoldLabel_PrintableASCII_SpecialChars(self): self._test_method([("~!@#^&*()_+=-[]{}:<>,./", "notporn")]) def test_AddGetGoldLabel_ExtendedASCIIChars(self): self._test_method([(u"™ž¤©", "notporn")]) def test_AddGetGoldLabel_UnicodeChars(self): self._test_method([(u"ૉେஇ", "notporn")])
class TestCommandStatus(unittest.TestCase): def setUp(self): self.tc = TroiaClient(ADDRESS) def tearDown(self): self.tc.delete() def test_GetRedirectData(self): self.tc.create(CATEGORIES) assigend_labels_response = self.tc.get_assigned_labels() time.sleep(0.5) self.assertLessEqual(self.tc.get_status(assigend_labels_response['redirect'])['result'], []) self.tc.await_completion(self.tc.post_assigned_labels(ASSIGNED_LABELS)) self.assertEqual(len(ASSIGNED_LABELS), len(self.tc.await_completion(self.tc.get_assigned_labels())['result'])) #still empty self.assertListEqual(self.tc.get_status(assigend_labels_response['redirect'])['result'], [])
class TestEvaluationLabels(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) response = self.client.create(CATEGORIES) self.assertEqual('OK', response['status']) def tearDown(self): self.client.delete() def _test_method(self, eval_data): response = self.client.await_completion(self.client.post_evaluation_objects(eval_data)) self.assertEqual('OK', response['status']) #get the unassigned labels response = self.client.await_completion(self.client.get_evaluation_objects()) self.assertEqual('OK', response['status']) result = response['result'] self.assertEqual(len(eval_data), len(result)) results = [(evaluationLabel['name'], evaluationLabel['evaluationLabel']) for evaluationLabel in response['result']] for evalLabel in eval_data: self.assertTrue(evalLabel in results) def test_AddGetEmptyEvaluationLabels(self): self._test_method([]) def test_AddGetEvaluationLabels_PrintableASCII_RegularChars(self): self._test_method(EVALUATION_DATA) def test_AddGetEvaluationLabel_PrintableASCII_SpecialChars(self): self._test_method([('~!@%#^&*()_+=-[]{}:<>,./', 'notporn')]) def test_AddGetEvaluationLabel_ExtendedASCIIChars(self): self._test_method([(u'™ž¤©', 'notporn')]) def test_AddGetEvaluationLabel_UnicodeChars(self): self._test_method([(u'ૉେஇ', 'notporn')]) def test_AddGetEvaluationLabel_LongLabelNames(self): response = self.client.await_completion(self.client.post_evaluation_objects([('sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs', 'notporn')])) self.assertEqual('ERROR', response['status']) self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result'])
class NominalSimulation(Simulation): def __init__(self, algorithm, *args, **kwargs): super(NominalSimulation, self).__init__(*args, **kwargs) self.algorithm = algorithm def create(self): self.tc = TroiaClient(TROIA_ADDRESS) categories, init_kwargs = self.data_generator.gen_init_data() self.tc.create(categories, algorithm=self.algorithm, **init_kwargs) def _upload_golds(self, golds): return ret_exectime(self.tc, self.tc.post_gold_data(golds)) def _upload_assigns_package(self, assigns): return ret_exectime(self.tc, self.tc.post_assigned_labels(assigns)) def compute(self): return ret_exectime(self.tc, self.tc.post_compute()) def shutdown(self): self.tc.delete()
class TestPrediction(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) self.client.create(CATEGORIES, costMatrix=COST_MATRIX, categoryPriors=CATEGORY_PRIORS, algorithm=self.algorithm) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_evaluation_objects(EVALUATION_DATA)) self.client.await_completion(self.client.post_compute()) def tearDown(self): self.client.delete() def _getObjectsPrediction(self, method, expectedResults): response = self.client.await_completion(self.client.get_objects_prediction(method)) self.assertEqual('OK', response['status']) for categories in response['result']: self.assertEqual(expectedResults[categories['objectName']], categories['categoryName']) def _getEstimatedObjectsCost(self, costMethod, expectedCosts): response = self.client.await_completion(self.client.get_estimated_objects_cost(costMethod)) self.assertEqual('OK', response['status']) for cost in response['result']: self.assertTrue(abs(expectedCosts[cost['objectName']] - cost['value']) / 100 < TOLERANCE) def _getEvaluatedObjectsCost(self, labelChoosingMethod, expectedResults): response = self.client.await_completion(self.client.get_evaluated_objects_cost(labelChoosingMethod)) self.assertEqual('OK', response['status']) for item in response['result']: self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE) def _getEstimatedObjectsQuality(self, costAlgorithm, expectedDataQuality): response = self.client.await_completion(self.client.get_estimated_objects_quality(costAlgorithm)) self.assertEqual('OK', response['status']) for dataQuality in response['result']: self.assertTrue(abs(expectedDataQuality[dataQuality['objectName']] - dataQuality['value']) / 100 < TOLERANCE) def _getEvaluatedObjectsQuality(self, labelChoosingMethod, expectedResults): response = self.client.await_completion(self.client.get_evaluated_objects_quality(labelChoosingMethod)) self.assertEqual('OK', response['status']) for item in response['result']: self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE) def _getEstimatedObjectsCostSummary (self, expectedObjectsCosts): response = self.client.await_completion(self.client.get_objects_cost_estimated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedObjectsCosts.items(): self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE) def _getEvaluatedObjectsCostSummary (self, expectedObjectsCosts): response = self.client.await_completion(self.client.get_objects_cost_evaluated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedObjectsCosts.items(): self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE) def _getEstimatedObjectsQualitySummary(self, expectedDataQuality): response = self.client.await_completion(self.client.get_objects_quality_estimated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedDataQuality.items(): self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE) def _getEvaluatedObjectsQualitySummary(self, expectedDataQuality): response = self.client.await_completion(self.client.get_objects_quality_evaluated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedDataQuality.items(): self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE) def _getEstimatedWorkerCost(self, costAlgorithm, expectedWorkerCost): response = self.client.await_completion(self.client.get_estimated_workers_cost(costAlgorithm)) self.assertEqual('OK', response['status']) for workerCost in response['result']: self.assertTrue(abs(expectedWorkerCost[workerCost['workerName']] - workerCost['value']) / 100 < TOLERANCE) def _getEstimatedWorkerQuality(self, costAlgorithm, expectedWorkerQuality): response = self.client.await_completion(self.client.get_estimated_workers_quality(costAlgorithm)) self.assertEqual('OK', response['status']) for workerQuality in response['result']: self.assertTrue(abs(expectedWorkerQuality[workerQuality['workerName']] - workerQuality['value']) / 100 < TOLERANCE) def _getEvaluatedWorkerQuality(self, costAlgorithm, expectedResults): response = self.client.await_completion(self.client.get_evaluated_workers_quality(costAlgorithm)) self.assertEqual('OK', response['status']) for item in response['result']: self.assertTrue(abs(expectedResults[item['workerName']] - item['value']) / 100 < TOLERANCE) def _getWorkersQualityEstimatedSummary(self, expectedWorkerQuality): response = self.client.await_completion(self.client.get_workers_quality_estimated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedWorkerQuality.items(): self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE) def _getWorkersQualityEvaluatedSummary(self, expectedWorkerQuality): response = self.client.await_completion(self.client.get_workers_quality_evaluated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedWorkerQuality.items(): self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE) def _getCategoryProbability(self, expectedProbabilities): for object in set((x[1] for x in ASSIGNED_LABELS)): response = self.client.await_completion(self.client.get_probability_distribution(object)) self.assertEqual('OK', response['status']) self.assertEqual(expectedProbabilities[object], response['result'])
class TestCategories(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) def tearDown(self): self.client.delete() def _test_method(self, categories): response = self.client.create(categories) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_categories()) self.assertEqual('OK', response['status']) for category in categories: self.assertTrue(category in response['result']) def _create_priors(self, categories, priors): return [{'categoryName': c, "value": p} for c, p in zip(categories, priors)] def test_AddGetCategories_LongCategoryNames(self): categories = [u'hjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdh', u'hjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhfhdfgjkshfghdsfkgjldkgjfdkgjdflgjfkdljajdghafdhjkdhsdsd'] response = self.client.create(categories) self.assertEqual('ERROR', response['status']) self.assertEqual('Category names should be shorter than 50 chars', response['result']) def test_AddGetCategories_PrintableASCII_SpecialChars(self): categories = [u'!@#$,.{}[]', u'2ndCategory'] self._test_method(categories) def test_AddGetCategories_ExtendedASCIIChars(self): categories = [u'œŒ', u'ÀÆË™ž¤©'] self._test_method(categories) def test_AddGetCategories_UnicodeChars(self): categories = [u'ૉେஇΨҖӖմڅ', u'ూഹܬआਖ਼'] self._test_method(categories) def test_CreateJobWithoutCategoryPriors_GetPriors(self): categories = [u'category1', u'category2'] response = self.client.create(categories) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_categories()) self.assertEqual('OK', response['status']) for category in categories: self.assertTrue(category in response['result']) def test_CreateJobWithCategoryPriors_GetPriors(self): categories = [u'category1', u'category2'] priors = [0.3, 0.7] response = self.client.create(categories, categoryPriors=self._create_priors(categories, priors)) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_categories()) self.assertEqual('OK', response['status']) for category in categories: self.assertTrue(category in response['result']) def test_CreateJobWithoutCostMatrix_2Categories_GetCostMatrix(self): categories = [u'category1', u'category2'] response = self.client.create(categories) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_cost_matrix()) self.assertEqual('OK', response['status']) #check that the default 0-1 cost matrix is created for matrix_value in response['result']: self.assertEqual(0.0 if matrix_value['from'] == matrix_value['to'] else 1.0, matrix_value['value']) def test_CreateJobWithoutCostMatrix_3Categories_GetCostMatrix(self): categories = [u'category1', u'category2', u'category3'] response = self.client.create(categories) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_cost_matrix()) self.assertEqual('OK', response['status']) #check that the default 0-1 cost matrix is created for matrix_value in response['result']: self.assertEqual(0.0 if matrix_value['from'] == matrix_value['to'] else 1.0, matrix_value['value']) def test_CreateJobWithCostMatrix_GetCostMatrix(self): categories = [u'category1', u'category2', u'category3'] cMatrix = [{"from": "category1", "to": "category1", "value": 0.0}, {"from": "category1", "to": "category2", "value": 0.5}, {"from": "category1", "to": "category3", "value": 0.5}, {"from": "category2", "to": "category1", "value": 0.5}, {"from": "category2", "to": "category2", "value": 0.0}, {"from": "category2", "to": "category3", "value": 0.5}, {"from": "category3", "to": "category1", "value": 0.5}, {"from": "category3", "to": "category2", "value": 0.5}, {"from": "category3", "to": "category3", "value": 0.0}] response = self.client.create(categories, costMatrix=cMatrix) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_cost_matrix()) self.assertEqual('OK', response['status']) for matrix_value in response['result']: self.assertEqual(0.0 if matrix_value['from'] == matrix_value['to'] else 0.5, matrix_value['value'])
class TestUnassignedLabels(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) def get_priors(self, categories): return [{'categoryName': c, "value": 1. / len(categories)} for c in categories] def tearDown(self): self.client.delete() def _test_method(self, categories, priors, unassignedLabels, expectedProbabilities): response = self.client.create(categories, categoryPriors=priors, algorithm="BMV") self.assertEqual('OK', response['status']) #post the unassigned labels response = self.client.await_completion(self.client.post_objects(unassignedLabels)) self.assertEqual('OK', response['status']) #get the unassigned labels response = self.client.await_completion(self.client.get_objects()) self.assertEqual('OK', response['status']) self.assertEqual(len(unassignedLabels), len(response['result'])) if (unassignedLabels): result = response['result'][0] self.assertEqual(unassignedLabels[0], result['name']) def test_AddGetObjects_EmptyLabels(self): self._test_method(CATEGORIES, CATEGORY_PRIORS, [], []) def test_AddGetUnassignedLabels_LongLabelNames(self): categories = ["category1", "category2"] priors = [{"categoryName": "category1", "value": 0.0000000001}, {"categoryName": "category2", "value": 0.9999999999}] unassignedLabels = ["sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs"] expectedProbabilities = [('category1', 0.5), ('category2', 0.5)] response = self.client.create(categories, categoryPriors=priors, algorithm="BMV") self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_objects(unassignedLabels)) self.assertEqual('ERROR', response['status']) self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result']) def test_AddGetUnassignedLabels_PrintableASCII_RegularChars(self): categories = ["category1", "category2", "category3", "category4"] unassignedLabels = ["testObject1"] expectedProbabilities = [('category1', 0.25), ('category2', 0.25), ('category3', 0.25), ('category4', 0.25)] self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities) def test_AddGetUnassignedLabels_PrintableASCII_SpecialChars(self): categories = ["category1", "category2", "category3"] unassignedLabels = ["~!@#%^&*()_+=-[]{}:<>,./"] expectedProbabilities = [('category1', 0.3333333333333333), ('category2', 0.3333333333333333), ('category3', 0.3333333333333333)] self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities) def test_AddGetUnassignedLabels_ExtendedASCIIChars(self): categories = ["category1", "category2"] unassignedLabels = [u"ëñµ¼Úæ"] expectedProbabilities = [('category1', 0.5), ('category2', 0.5)] self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities) def test_AddGetUnassignedLabels_UnicodeChars(self): categories = ["category1", "category2"] unassignedLabels = [u"ూഹܬआਖ਼"] expectedProbabilities = [('category1', 0.5), ('category2', 0.5)] self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities) def test_GetObjectAssigns(self): response = self.client.create(CATEGORIES) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.assertEqual('OK', response['status']) #post the unassigned label unassignedLabel = [u"ూഹܬआਖ਼"] response = self.client.await_completion(self.client.post_objects(unassignedLabel)) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_objects()) self.assertEqual('OK', response['status']) result = response['result'] self.assertEqual(6, len(result)) results = [] for object in result: if object['name'] not in unassignedLabel: assigns = self.client.await_completion(self.client.get_object_assigns(object['name']))['result'] for a in assigns: results.append((a['worker'], a['object'], a['label'])) for label in ASSIGNED_LABELS: self.assertTrue(label in results) def test_GetObjectInfo(self): response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_assigned_labels([('worker1', 'url1', 'p**n'), ('worker1', 'url2', 'p**n')])) self.assertEqual('OK', response['status']) #post the unassigned label objects_without_assigns = ["newUnassignedLabel"] response = self.client.await_completion(self.client.post_objects(objects_without_assigns)) self.assertEqual('OK', response['status']) #get labels response = self.client.await_completion(self.client.get_objects()) self.assertEqual('OK', response['status']) result = response['result'] self.assertEqual(3, len(result)) self.client.await_completion(self.client.post_compute()) for label in result: response = self.client.await_completion(self.client.get_probability_distribution(label['name'])) dist = response['result'][0] self.assertAlmostEqual(dist['value'], 0.5 if label['name'] in objects_without_assigns else 0.9 if dist['categoryName'] == 'p**n' else 0.1)
class TestWorkers(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) def tearDown(self): self.client.delete() def test_AddGetEmptyWorkers(self): self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS) self.client.await_completion(self.client.post_compute()) response = self.client.await_completion(self.client.get_workers()) self.assertEqual('OK', response['status']) self.assertEqual([], response['result']) def test_AddGetWorkers_BeforeCompute(self): self.client.create(CATEGORIES) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES)) response = self.client.await_completion(self.client.get_workers()) self.assertEqual(5, len(response['result'])) for w in response['result']: self.assertEqual(5, w['value']['assigns']) def test_AddGetWorkers_AfterCompute(self): self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_compute()) #assigns check response = self.client.await_completion(self.client.get_workers()) self.assertEqual(5, len(response['result'])) for w in response['result']: self.assertEqual(5, w['value']['assigns']) #confusion matrices check response = self.client.await_completion(self.client.get_workers_confusion_matrix()) exp = { u'worker1': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, {u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}], u'worker3': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, {u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}], u'worker2': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.}, {u'to': u'p**n', u'from': u'notporn', u'value': 0.3333}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.6666}], u'worker5': [{u'to': u'p**n', u'from': u'p**n', u'value': 0.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 1.0}, {u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}], u'worker4': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, {u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}]} for w in response['result']: worker_name = w['workerName'] for e1 in w['value']: exists = False for e2 in exp[worker_name]: if e2['to'] == e1['to'] and e2['from'] == e1['from']: exists = True self.assertAlmostEqual( e2['value'], e1['value'], 2, "{}, from: {}, to: {}. expected:{}, was: {}".format(worker_name, e1['from'], e1['to'], e2['value'], e1['value'])) self.assertTrue(exists) def test_GetWorkerInfo(self): self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES)) self.client.await_completion(self.client.post_compute()) response = self.client.await_completion(self.client.get_worker_info('worker1')) self.assertTrue(5, response['result']['value']['assigns']) self.assertEqual(2, response['result']['value']['goldTests']) self.assertEqual(1, response['result']['value']['correctGoldTests']) self.assertTrue('worker1', response['result']['workerName']) def test_GetWorkerAssigns(self): self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_compute()) response = self.client.await_completion(self.client.get_worker_assigns('worker1')) print response self.assertTrue(5, len(response['result'])) for a in response['result']: self.assertTrue((a['worker'], a['object'], a['label']) in ASSIGNED_LABELS)
class TestCachedScheduler(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) self.scheduler = 'CachedScheduler' self.utils = TestUtils() def tearDown(self): self.client.delete() def _createTestPrereq(self, algorithm, scheduler, calculator, assigns, categories = CATEGORIES, categoryPriors = CATEGORY_PRIORS, costMatrix = COST_MATRIX): response = self.client.create( categories, categoryPriors=categoryPriors, algorithm=algorithm, scheduler=scheduler, prioritycalculator=calculator, costMatrix=costMatrix ) self.assertEqual('OK', response['status']) self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(assigns))['status']) self.assertEqual('OK', self.client.await_completion(self.client.post_compute())['status']) def getObjectCountsList(self, assigns, reverse=False, excludedObjectsList=None): #create the dictionary containing the objects and the associated no of assigns labelsDict = {} for l in assigns: if l[1] in labelsDict.keys(): labelsDict[l[1]] += 1 else: labelsDict[l[1]] = 1 #sort the objects ascending, based on the no of labels sortedList = sorted(labelsDict.items(), key=itemgetter(1), reverse=reverse) if (excludedObjectsList): for value in sortedList: if value[0] in excludedObjectsList: sortedList.remove(value) return sortedList def getObjectCostsList(self): response = self.client.await_completion(self.client.get_estimated_objects_cost("ExpectedCost")) self.assertEquals("OK", response['status']) objectCosts = {} for result in response['result']: objectCosts[result['objectName']] = result['value'] #sort the objects descending, based on cost objectCostList = sorted(objectCosts.items(), key=itemgetter(1), reverse=True) return objectCostList def getAssignedLabels(self): response = self.client.await_completion(self.client.get_assigned_labels()) self.assertEqual('OK', response['status']) assignedLabels = [(l['worker'], l['object'], l['label']) for l in response['result']] return assignedLabels def _runScheduler(self, workerId=None): if (workerId): return self.client.get_next_worker_object(workerId) else: return self.client.get_next_object() def _runTestMethod(self, calculator, expectedObjectList, newAssign, workerId=None, excludedObjectsList=None): for (_, objectCost) in expectedObjectList: response = self.client.await_completion(self._runScheduler(workerId)) objectName = response['result']['name'] #get the objects with equal costs equalCostObjects = [item[0] for item in expectedObjectList if item[1] == objectCost] self.assertTrue(objectName in equalCostObjects) # This one should be null. That means the 'result' key is not present in the response. response = self.client.await_completion(self._runScheduler(workerId)) self.assertIsNone(response.get('result', None)) # Add assign to the object. The object should be returned by subsequent 'nextObject' call. self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(newAssign))['status']) if calculator == 'countassigns': newObjectsList = self.getObjectCountsList(newAssign, False, excludedObjectsList) else: self.assertEqual('OK', self.client.await_completion(self.client.post_compute())['status']) newObjectsList = self.getObjectCostsList() #get the objects having the maximum priority maxPriorityObjects = [item[0] for item in newObjectsList if item[1] == newObjectsList[0][1]] response = self.client.await_completion(self._runScheduler(workerId)) self.assertEqual('OK',response['status']) if response.get('result') == None: self.assertTrue(len(newAssign) == 0) else: self.assertTrue(response['result']['name'] in maxPriorityObjects) @unittest.skip('Skipping test') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddNewAssigns(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'notporn'), ('worker1', 'object2', 'p**n'), ('worker2', 'object2', 'notporn'), ('worker3', 'object2', 'p**n')] expectedObjectsList = self.getObjectCountsList(assigns, False) newAssigns = [('worker1', 'object3', 'p**n')] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, newAssigns) @unittest.skip('Skipping test') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddEmptyAssign(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker1', 'object2', 'p**n'), ('worker2', 'object2', 'p**n')] expectedObjectsList = self.getObjectCountsList(assigns, False) newAssign = [] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, newAssign) @unittest.skip('Skipping test') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextObject_SameLabelCounts_AddNewAssign(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'p**n'), ('worker1', 'object2', 'p**n'), ('worker2', 'object2', 'p**n')] expectedObjectsList = self.getObjectCountsList(assigns, False) newAssign = [('worker3', 'object0', 'notporn')] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, newAssign) @unittest.skip('Skipping test') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextObject_SameLabelCounts_AddEmptyAssign(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'p**n'), ('worker1', 'object2', 'p**n'), ('worker2', 'object2', 'p**n')] expectedObjectsList = self.getObjectCountsList(assigns, False) newAssign = [] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, newAssign) @unittest.skip('Skipping test') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CostBasedCalculator_GetNextObject_SameObjectCosts(self, algorithm): calculator = 'costbased' categories = ["cat1", "cat2"] categoryPriors = [{"categoryName": "cat1", "value": 0.5}, {"categoryName": "cat2", "value": 0.5}] costMatrix = [{"from": "cat1", "to": "cat2", "value": 1.0}, {"from": "cat1", "to": "cat1", "value": 0.0}, {"from": "cat2", "to": "cat1", "value": 1.0}, {"from": "cat2", "to": "cat2", "value": 0.0}] assigns = [('worker1', 'object1', 'cat1'), ('worker2', 'object1', 'cat1'), ('worker3', 'object2', 'cat1'), ('worker4', 'object2', 'cat1')] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors, costMatrix) objectCostList = self.getObjectCostsList() newAssign = [('worker3', 'object0', 'cat1')] self._runTestMethod(calculator, objectCostList, newAssign) @unittest.skip('Skipping test') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CostBasedCalculator_GetNextObject_DifferentObjectCosts(self, algorithm): calculator = 'costbased' categories = ["p**n", "notporn"] categoryPriors = [{"categoryName": "p**n", "value": 0.1}, {"categoryName": "notporn", "value": 0.9}] costMatrix = [{"from": "p**n", "to": "notporn", "value": 1.0}, {"from": "p**n", "to": "p**n", "value": 0.0}, {"from": "notporn", "to": "p**n", "value": 1.0}, {"from": "notporn", "to": "notporn", "value": 0.0}] assigns = [('worker0', 'object0', 'notporn'), ('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'p**n')] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors, costMatrix) objectCostList = self.getObjectCostsList() newAssign = [('worker3', 'object2', 'p**n')] self._runTestMethod(calculator, objectCostList, newAssign) @unittest.skip('Skipping test') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CostBasedCalculator_GetNextObject_DifferentObjectCosts_AddEmptyLabel(self, algorithm): calculator = 'costbased' categories = ["p**n", "notporn"] categoryPriors = [{"categoryName": "p**n", "value": 0.1}, {"categoryName": "notporn", "value": 0.9}] costMatrix = [{"from": "p**n", "to": "notporn", "value": 1.0}, {"from": "p**n", "to": "p**n", "value": 0.0}, {"from": "notporn", "to": "p**n", "value": 1.0}, {"from": "notporn", "to": "notporn", "value": 0.0}] assigns = [('worker0', 'object0', 'notporn'), ('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'p**n')] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors, costMatrix) objectCostList = self.getObjectCostsList() newAssign = [] self._runTestMethod(calculator, objectCostList, newAssign) @unittest.skip('Skipping test') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextWorkerObject_SameLabelCounts(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker2', 'object2', 'notporn'), ('worker2', 'object3', 'notporn'), ('worker3', 'object2', 'notporn'), ('worker3', 'object3', 'notporn') ] expectedObjectsList = [('object2', 2), ('object3', 2)] newAssign = [('worker4', 'object2', 'notporn')] excludedObjectsList = ['object1'] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, newAssign, 'worker1', excludedObjectsList) @unittest.skip('Skipping test') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextWorkerObject_DifferentLabelCounts(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'p**n'), ('worker2', 'object2', 'notporn'), ('worker3', 'object2', 'notporn'), ('worker3', 'object3', 'p**n')] expectedObjectsList = [('object3', 1), ('object2', 2)] newAssign = [('worker4', 'object2', CATEGORIES[1])] excludedObjectsList = ['object1'] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, newAssign, 'worker1', excludedObjectsList)
class TestNormalScheduler(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) self.scheduler = 'normalscheduler' self.utils = TestUtils() def tearDown(self): self.client.delete() def _createTestPrereq(self, algorithm, scheduler, calculator, assigns, categories = CATEGORIES, categoryPriors = CATEGORY_PRIORS): response = self.client.create( categories, categoryPriors=categoryPriors, algorithm=algorithm, scheduler=scheduler, prioritycalculator=calculator ) self.assertEqual('OK', response['status']) self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(assigns))['status']) self.assertEqual('OK', self.client.await_completion(self.client.post_compute())['status']) def getObjectCountsList(self, assigns, reverse=False, excludedObjectsList=False): #create the dictionary containing the objects and the associated no of assigns labelsDict = {} for l in assigns: if l[1] in labelsDict.keys(): labelsDict[l[1]] += 1 else: labelsDict[l[1]] = 1 #sort the objects ascending, based on the no of labels sortedList = sorted(labelsDict.items(), key=itemgetter(1), reverse=reverse) if (excludedObjectsList): for value in sortedList: if value[0] in excludedObjectsList: sortedList.remove(value) return sortedList def getObjectCostsList(self): response = self.client.await_completion(self.client.post_compute()) self.assertEquals("OK", response['status']) response = self.client.await_completion(self.client.get_estimated_objects_cost("ExpectedCost")) self.assertEquals("OK", response['status']) objectCosts = {} for result in response['result']: objectCosts[result['objectName']] = result['value'] #sort the objects descending, based on cost objectCostList = sorted(objectCosts.items(), key=itemgetter(1), reverse=True) return objectCostList def getAssignedLabels(self): response = self.client.await_completion(self.client.get_assigned_labels()) self.assertEqual('OK', response['status']) assignedLabels = [(l['worker'], l['object'], l['label']) for l in response['result']] return assignedLabels def _runScheduler(self, workerId=None): if (workerId): return self.client.get_next_worker_object(workerId) else: return self.client.get_next_object() def _runTestMethod(self, calculator, expectedObjectList, newAssign, workerId=None, excludedObjectsList=None): for i in xrange(len(expectedObjectList)): response = self.client.await_completion(self._runScheduler(workerId)) self.assertTrue(response['result']['name'] in expectedObjectList) # Add assign to the object. The object should be returned by subsequent 'nextObject' call. self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(newAssign))['status']) response = self.client.await_completion(self._runScheduler(workerId)) self.assertEqual('OK',response['status']) if calculator == 'countassigns': newObjectsList = self.getObjectCountsList(self.getAssignedLabels(), False, excludedObjectsList) else: newObjectsList = self.getObjectCostsList() #get the objects having the maximum priority maxPriorityObjects = [item[0] for item in newObjectsList if item[1] == newObjectsList[0][1]] response = self.client.await_completion(self._runScheduler(workerId)) self.assertEqual('OK',response['status']) self.assertTrue(response['result']['name'] in maxPriorityObjects) @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddEmptyAssigns(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'notporn'), ('worker2', 'object2', 'notporn')] sortedList = self.getObjectCountsList(assigns, False) minValue = sortedList[0][1] expectedObjectsList = [o[0] for o in sortedList if o[1] == minValue] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, []) @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddNewAssigns(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'p**n'), ('worker1', 'object2', 'p**n'), ('worker2', 'object2', 'notporn'), ('worker3', 'object2', 'p**n'), ] sortedList = self.getObjectCountsList(assigns, False) minValue = sortedList[0][1] expectedObjectsList = [o[0] for o in sortedList if o[1] == minValue] newAssign = [('worker1', 'object3', 'p**n')] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, newAssign) @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextObject_SameLabelCounts(self, algorithm): calculator = 'countassigns' categories = ["cat1", "cat2"] categoryPriors = [{"categoryName": "cat1", "value": 0.5}, {"categoryName": "cat2", "value": 0.5}] assigns = [('worker1', 'object1', 'cat1'), ('worker1', 'object2', 'cat1'), ('worker2', 'object1', 'cat2'), ('worker2', 'object2', 'cat2')] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors) sortedList = self.getObjectCountsList(assigns, False) minValue = sortedList[0][1] expectedObjectList = [o[0] for o in sortedList if o[1] == minValue] newAssign = [('worker3', 'object3', 'cat1')] self._runTestMethod(calculator, expectedObjectList, newAssign) @data('BDS', 'IDS', 'BMV', 'IMV') def test_CostBasedCalculator_GetNextObject_SameCosts(self, algorithm): calculator = 'costbased' categories = ["cat1", "cat2"] categoryPriors = [{"categoryName": "cat1", "value": 0.5}, {"categoryName": "cat2", "value": 0.5}] assigns = [('worker1', 'object1', 'cat1'), ('worker2', 'object1', 'cat1'), ('worker3', 'object2', 'cat1'), ('worker4', 'object2', 'cat1')] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors) sortedList = self.getObjectCostsList() minValue = sortedList[0][1] expectedObjectList = [o[0] for o in sortedList if o[1] == minValue] newAssign = [('worker5', 'object3', 'cat1')] self._runTestMethod(calculator, expectedObjectList, newAssign) @data('BDS', 'IDS', 'BMV', 'IMV') def test_CostBasedCalculator_GetNextObject_DifferentCosts(self, algorithm): calculator = 'costbased' categories = ["cat1", "cat2", "cat3"] categoryPriors = [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.3}, {"categoryName": "cat3", "value": 0.6}] assigns = [('worker1', 'object1', 'cat1'), ('worker2', 'object1', 'cat2'), ('worker2', 'object2', 'cat1'), ('worker3', 'object1', 'cat1'), ('worker3', 'object2', 'cat2')] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors) sortedList = self.getObjectCostsList() minValue = sortedList[0][1] expectedObjectList = [o[0] for o in sortedList if o[1] == minValue] newAssign = [('worker4', 'object0', 'cat3')] self._runTestMethod(calculator, expectedObjectList, newAssign) @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextWorkerObject_DifferentLabelCounts(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'p**n'), ('worker2', 'object2', 'notporn'), ('worker3', 'object2', 'notporn'), ('worker4', 'object2', 'notporn'), ('worker3', 'object3', 'p**n') ] expectedObjectsList = ['object3'] newAssigns = [('worker4', 'object3', CATEGORIES[0])] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, newAssigns, 'worker1') @data('BDS', 'IDS', 'BMV', 'IMV') def test_CountAssignsCalculator_GetNextWorkerObject_SameLabelCounts(self, algorithm): calculator = 'countassigns' assigns = [('worker1', 'object1', 'p**n'), ('worker2', 'object1', 'p**n'), ('worker2', 'object2', 'notporn'), ('worker3', 'object2', 'p**n'), ('worker3', 'object3', 'notporn'), ('worker4', 'object3', 'p**n') ] expectedObjectsList = ['object3', 'object2'] newAssigns = [('worker4', 'object4', CATEGORIES[0])] self._createTestPrereq(algorithm, self.scheduler, calculator, assigns) self._runTestMethod(calculator, expectedObjectsList, newAssigns, 'worker1')
class TestJobs(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) def tearDown(self): self.client.delete() def assertJobData(self, response, expAlgorithm, expNoAssigns, expNoGoldObjects, expNoObjects, expNoWorkers): self.assertEqual('OK', response['status']) self.assertEqual(expAlgorithm, response['result']['initializationData']['algorithm']) self.assertEqual(expNoAssigns, response['result']['assigns']) self.assertEqual(expNoGoldObjects, response['result']['goldObjects']) self.assertEqual(expNoObjects, response['result']['objects']) self.assertEqual(expNoWorkers, response['result']['workers']) def test_createJob_NoJobType(self): response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS) response = self.client.await_completion(self.client.get_job_status()) self.assertJobData(response, 'BDS', 0, 0, 0, 0) @data('BDS', 'bdS', 'IDS', 'iDS', 'BMV', 'ImV') def test_createJob(self, algorithm): response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, algorithm=algorithm) self.assertEqual('OK', response['status']) self.assertTrue('New job created with ID: RANDOM_' in response['result']) response = self.client.await_completion(self.client.get_job_status()) self.assertJobData(response, algorithm, 0, 0, 0, 0) @data(' bDS', 'BdS ', ' BDS ') def test_createJob_TrimParams(self, algorithm): response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, algorithm=algorithm) self.assertEqual('ERROR', response['status']) self.assertEqual('Unknown algorithm type: [%s]. Did you mean: [bds]' %(algorithm), response['result']) def test_createJob_WrongJobType(self): response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, algorithm='test') self.assertEqual('ERROR', response['status']) self.assertTrue('Unknown algorithm type' in response['result']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_NoCategories(self, algorithm): response = self.client.create(None, algorithm=algorithm) self.assertEqual('ERROR', response['status']) self.assertEqual('There is no categories collection', response['result']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_EmptyCategories(self, algorithm): response = self.client.create([], algorithm=algorithm) self.assertEqual('ERROR', response['status']) self.assertEqual('There should be at least two categories', response['result']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_2Categories_SameCategoryNames(self, algorithm): categories = [u'category1', u'category1'] response = self.client.create(categories, algorithm=algorithm) self.assertEqual('ERROR', response['status']) self.assertEqual('Category names should be different', response['result']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_3Categries_SameCategoryNames(self, algorithm): categories = [u'category1', u'category1', u'category2'] response = self.client.create(categories, algorithm=algorithm) self.assertEqual('ERROR', response['status']) self.assertEqual('Category names should be different', response['result']) def get_priors(self, categories, priors): return [{'categoryName': c, "value": p} for c, p in zip(categories, priors)] @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_SumOfPriorsLessThanOne_NoCostMatrix(self, algorithm): categories = ["p**n", "notporn"] priors = [0.3, 0.5] response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), algorithm=algorithm) self.assertEqual('ERROR', response['status']) self.assertEqual('Priors should sum up to 1. or not to be given (therefore we initialize the priors to be uniform across classes)', response['result']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_SumOfPriorsGreaterThanOne_NoCostMatrix(self, algorithm): categories = ["p**n", "notporn"] priors = [0.6, 0.5] response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), algorithm=algorithm) self.assertEqual('ERROR', response['status']) self.assertEqual('Priors should sum up to 1. or not to be given (therefore we initialize the priors to be uniform across classes)', response['result']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_SumOfPriorsEqualsOne_NoCostMatrix(self, algorithm): categories = ["p**n", "notporn"] priors = [0.234, 0.766] response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), algorithm=algorithm) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_job_status()) self.assertJobData(response, algorithm, 0, 0, 0, 0) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_SumOfPriorsLessThanOne_WithCostMatrix(self, algorithm): categories = ["p**n", "notporn"] priors = [0.3, 0.5] response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), costMatrix=COST_MATRIX, algorithm=algorithm) self.assertEqual('ERROR', response['status']) self.assertEqual('Priors should sum up to 1. or not to be given (therefore we initialize the priors to be uniform across classes)', response['result']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_SumOfPriorsGreaterThanOne_WithCostMatrix(self, algorithm): categories = ["p**n", "notporn"] priors = [0.6, 0.5] response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), costMatrix=COST_MATRIX, algorithm=algorithm) self.assertEqual('ERROR', response['status']) self.assertEqual('Priors should sum up to 1. or not to be given (therefore we initialize the priors to be uniform across classes)', response['result']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_SumOfPriorsEqualsOne_WithCostMatrix(self, algorithm): categories = ["p**n", "notporn"] priors = [0.1234, 0.8766] response = self.client.create(categories, categoryPriors=self.get_priors(categories, priors), costMatrix=COST_MATRIX, algorithm=algorithm) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_job_status()) self.assertJobData(response, algorithm, 0, 0, 0, 0) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_NoPriors_NoCostMatrix(self, algorithm): categories = ["p**n", "notporn"] response = self.client.create(categories, algorithm=algorithm) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_job_status()) self.assertJobData(response, algorithm, 0, 0, 0, 0) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_NoPriors_WithCostMatrix(self, algorithm): categories = ['p**n', 'notporn'] response = self.client.create(categories, costMatrix=COST_MATRIX, algorithm=algorithm) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_job_status()) self.assertJobData(response, algorithm, 0, 0, 0, 0) @data('BDS', 'IDS', 'BMV', 'IMV') def test_createJob_CostMatrixContainsNotExistingCategories(self, algorithm): categories = [u'category1', u'category2'] response = self.client.create(categories, costMatrix=COST_MATRIX, algorithm=algorithm) self.assertEqual('ERROR', response['status']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_deleteJob_ExistingJobId(self, algorithm): response = self.client.create(CATEGORIES, algorithm=algorithm) self.assertEqual('OK', response['status']) self.assertEqual('New job created with ID: ' + self.client.jid, response['result']) response = self.client.delete() self.assertEqual('OK', response['status']) self.assertEqual('Removed job with ID: ' + self.client.jid, response['result']) @data('BDS', 'IDS', 'BMV', 'IMV') def test_deleteJob_NonExistingJobId(self, algorithm): response = self.client.create(CATEGORIES, algorithm=algorithm) self.assertEqual('OK', response['status']) self.assertEqual('New job created with ID: ' + self.client.jid, response['result']) response = self.client.delete('NotExistingJob') self.assertEqual('ERROR', response['status']) self.assertEqual('Job with ID NotExistingJob does not exist', response['result'])
class TestAssignedLabels(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) def tearDown(self): self.client.delete() def _test_method(self, assigned_labels): #post the assigned labels response = self.client.await_completion(self.client.post_assigned_labels(assigned_labels)) self.assertEqual('OK', response['status']) #get the assigned labels response = self.client.await_completion(self.client.get_assigned_labels()) self.assertEqual('OK', response['status']) self.assertEqual(len(response['result']), len(assigned_labels)) results = [(l['worker'], l['object'], l['label']) for l in response['result']] for label in assigned_labels: self.assertTrue(label in results) def test_AddGetEmptyAssignedLabels(self): response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX) self.assertEqual('OK', response['status']) self._test_method([]) def test_AddAssignedLabelsWithInvalidCategory(self): categories = ["category1", "category2"] response = self.client.create(categories) self.assertEqual('OK', response['status']) #post the assigned labels assignedLabels = [('worker1', 'url1', 'category3')] response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels)) self.assertEqual('ERROR', response['status']) #get the assigned labels response = self.client.await_completion(self.client.get_assigned_labels()) self.assertEqual('OK', response['status']) result = response['result'] self.assertFalse(result) def test_AddGetAssignedLabels_LongWorkerNames(self): categories = ['category1', "category2"] response = self.client.create(categories) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_assigned_labels([('hjkdhfhdfgjkshfghdsgffgfhfghgjhghjgjgjgjgjgjgjldkgjhjkdhfhdfgjkshfghdsgffgfhfghgjhghjgjgjgjgjgjgjldkg', 'object_dsjgfhgfhgfhhjdfhgkgkhjkfklsdjfkljsdfj', 'category1')])) self.assertEqual('ERROR', response['status']) self.assertEqual('Internal error: Worker name should be shorter than 100 chars', response['result']) def test_AddGetAssignedLabels_LongObjectNames(self): categories = ['category1', "category2"] response = self.client.create(categories) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_assigned_labels([('hjkdhfhdfgjkshfghdsgffgfhfghgjhghjgkgj', 'object_dsjgfhgfhgfhhjdfhgjghghkgkhjkfklsdjfkljsdfjobjectdsjgfhgfhgfhhjdfhgjghghkgkhjkfklsdjfkljsdfjwd', 'category1')])) self.assertEqual('ERROR', response['status']) self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result']) def test_AddGetAssignedLabels_PrintableASCII_RegularChars(self): response = self.client.create(CATEGORIES) self.assertEqual('OK', response['status']) self._test_method(ASSIGNED_LABELS) def test_AddGetAssignedLabels_PrintableASCII_SpecialChars(self): categories = ['category1', "category2", "category3"] response = self.client.create(categories) self.assertEqual('OK', response['status']) self._test_method([('a%a', '~!@#%^&*()_+=-[]{}:<>,./', 'category1'), ('b%%b', '%%%', 'category2'), ('c%%!<>c', '~!@#^&*[](){}-_+=<>?/.,:', 'category3')]) def test_AddGetAssignedLabels_ExtendedASCIIChars(self): categories = ['category1', "category2", "category3"] response = self.client.create(categories) self.assertEqual('OK', response['status']) self._test_method([(u'®¶', u'œŒ', 'category1'), (u'ÀÆË', u'™ž¤©', 'category2'), (u'ëñ', u'µ¼Úæ', 'category3')]) def test_AddGetAssignedLabels_UnicodeChars(self): categories = ['category1', "category2", "category3"] response = self.client.create(categories) self.assertEqual('OK', response['status']) self._test_method([(u'ૉେஇ', u'ΨҖӖմڅ', 'category1'), (u'ూഹ', u'ܬआਖ਼', 'category2')])
class TestConfiguration(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) self.initialConfigParams = self._get_config_params() def tearDown(self): #revert any of the config changes response = self.client.post_config(self.initialConfigParams) self.assertEqual(200, response.status_code) self.client.delete() def _get_config_params(self): response = self.client.get_config() htmlPage = BeautifulSoup(response.content) configParams = {} configParams['JOBS_STORAGE'] = htmlPage.find_all("option", {'selected':'selected'})[0]['value'] for inputTag in htmlPage.find_all("input"): try: configParams[inputTag['name']] = inputTag['value'] except: pass return configParams def test_Status(self): response = self.client.status() self.assertEqual('OK', response['status']) self.assertEqual('OK', response['result']['status']) #self.assertEqual('OK', response['result']['job_storage_status']) @unittest.skip('Skipping until the test is fixed') def test_GetDefaultConfig(self): response = self.client.get_config() self.assertEqual(200, response.status_code) @unittest.skip('Skipping until the test is fixed') def test_SetWrongDbUrl(self): inputParams = {'JOBS_STORAGE':'DB_FULL', 'DB_DRIVER_CLASS':'com.mysql.jdbc.Driver', 'DB_URL':'dbUrl', 'DB_NAME':'Troia' } response = self.client.post_config(inputParams) self.assertEqual(500, response.status_code) self.assertEqual(u'No suitable driver found for dbUrl?useUnicode=true&characterEncoding=utf-8', response.text) @unittest.skip('Skipping until the test is fixed') def test_SetWrongDbAccessSettings(self): inputParams = {'JOBS_STORAGE':'DB_FULL', 'DB_DRIVER_CLASS':'com.mysql.jdbc.Driver', 'DB_URL':'jdbc:mysql://localhost/', 'DB_NAME':'Troia', 'DB_PASSWORD':'******', 'DB_USER':'******', 'DOWNLOADS_PATH':'dbDownloadPath'} response = self.client.post_config(inputParams) self.assertEqual(500, response.status_code) self.assertEqual(u'Access denied for user \'dbUser\'@\'localhost\' (using password: YES)', response.text) @unittest.skip('Skipping until the test is fixed') def test_SetConfigParams(self): inputParams = {'CACHE_DUMP_TIME':'600', 'CACHE_SIZE':'100', 'DB_DRIVER_CLASS':'com.mysql.jdbc.Driver', 'DB_NAME':'Troia', 'DB_PASSWORD':'******', 'DB_URL':'jdbc:mysql://localhost/', 'DB_USER':'******', 'DOWNLOADS_PATH':'dbDownloadPath', 'EXECUTOR_THREADS_NUM':'10', 'FREEZE_CONFIGURATION_AT_START':'false', 'MEMCACHE_EXPIRATION_TIME':'100', 'MEMCACHE_PORT':'30', 'MEMCACHE_URL':'memCacheUrl', 'RESPONSES_CACHE_SIZE':'10', 'RESPONSES_DUMP_TIME':'100', 'JOBS_STORAGE':'DB_FULL'} response = self.client.post_config(inputParams) self.assertEqual(200, response.status_code) currentConfigParams = self._get_config_params() for key in inputParams.keys(): self.assertEqual(inputParams[key], currentConfigParams[key]) @unittest.skip('Skipping until the test is fixed') def test_ResetDB(self): response = self.client.resetDB() self.assertEqual(200, response.status_code)
class TestQualitySensitivePayments(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) def tearDown(self): self.client.delete() def testQSP_ZeroWorkerQUality(self): assignedLabels = [ ('worker1', 'url1', 'cat2'), ('worker1', 'url2', 'cat1'), ('worker2', 'url1', 'cat1'), ('worker2', 'url2', 'cat1'), ('worker3', 'url1', 'cat1'), ('worker3', 'url2', 'cat2')] goldLabels = [ ('url1', 'cat1'), ('url2', 'cat2') ] response = self.client.create(["cat1", "cat2"]) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels)) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_gold_data(goldLabels)) self.assertEqual('OK', response['status']) response = self.client.post_compute() self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost")) self.assertEqual('OK', response['status']) pprint(response) response = self.client.await_completion(self.client.get_workers_quality_payment()) pprint(response) def testQSP_NegativeWorkerQuality(self): assignedLabels = [ ('worker1', 'url1', 'cat2'), ('worker1', 'url2', 'cat1'), ('worker1', 'url3', 'cat2'), ('worker2', 'url1', 'cat1'), ('worker2', 'url2', 'cat1'), ('worker2', 'url3', 'cat1'), ('worker3', 'url1', 'cat1'), ('worker3', 'url2', 'cat2'), ('worker3', 'url3', 'cat1') ] goldLabels = [ ('url1', 'cat1'), ('url2', 'cat2'), ('url3', 'cat1') ] response = self.client.create(["cat1", "cat2"], categoryPriors= [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.9}]) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels)) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_gold_data(goldLabels)) self.assertEqual('OK', response['status']) response = self.client.post_compute() self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_workers_confusion_matrix()) pprint(response) response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost")) self.assertEqual('OK', response['status']) pprint(response) response = self.client.await_completion(self.client.get_workers_quality_payment()) pprint(response)
from client.gal import TroiaClient from nominalJobsTests.testSettings import * def check_status(res): if res['status'] != "OK" or res['result']['job_storage_status'] != "OK": print "FAILURE" if __name__ == "__main__": client = TroiaClient(ADDRESS) check_status(client.status()) client.create(CATEGORIES) client.await_completion(client.post_assigned_labels(ASSIGNED_LABELS)) client.await_completion(client.post_gold_data(GOLD_SAMPLES)) client.await_completion(client.post_compute()) # client.await_completion(client.get_predictions_objects()) # client.await_completion(client.get_prediction_workers_quality()) client.await_completion(client.get_prediction_zip()) client.delete() check_status(client.status())
class BugRegressionTests(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) def tearDown(self): self.client.delete() def test_GH_83(self): """ Test for github issue https://github.com/ipeirotis/Troia-Server/issues/83 Thanks to PeccatorImpius for reporting """ categories = ['yes', 'no', 'blank'] response = self.client.create(categories, prioritycalculator="CostBased", costMatrix=[{"value":5.0, "to":"blank", "from":"blank"}, {"value":5.0, "to":"no", "from":"blank"}, {"value":5.0, "to":"yes", "from":"blank"}, {"value":1.0, "to":"blank", "from":"no"}, {"value":0.0, "to":"no", "from":"no"}, {"value":1.0, "to":"yes", "from":"no"}, {"value":1.0, "to":"blank", "from":"yes"}, {"value":1.0, "to":"no", "from":"yes"}, {"value":0.0, "to":"yes", "from":"yes"}], epsilon=1.0E-4, iterations=10, algorithm='IDS', scheduler='NormalScheduler') self.assertEqual('OK', response['status']) assignedLabels = [ ('A2G061ZJVQXGE', '21K74J79KQW6KZ5ZI8OFKD455U4HJ6', 'blank'), ('A2G061ZJVQXGE', '2FWPLSP75KLMPN1Y0O8QMQVJZMCZTJ', 'yes'), ('A224TK7J4KA1LV', '283PHIT3HVWAEB01E1AQA1727RZCKR', 'yes'), ('A224TK7J4KA1LV', '2S3YHVI44OS23CC3P15Z0BAPDTU4Y4', 'yes'), ('A224TK7J4KA1LV', '2NCXFCD45XCECEVNZ7BIV3TGXMSYWJ', 'yes'), ('A224TK7J4KA1LV', '295QQ7Q670519BUI3FTTYC1XUY7WC6', 'no'), ('A224TK7J4KA1LV', '255NVRH1KHO9X796JKEVHLKWFFDZQ5', 'yes'), ('A224TK7J4KA1LV', '21Z1Q6SVQ8H8RD8PIZWOJVCI99NN4U', 'yes'), ('A224TK7J4KA1LV', '2JGU98JHSTLBMCIANX0IWBJER94EQE', 'yes'), ('A224TK7J4KA1LV', '2Z3KH1Q6SVQ80ZQH28XLDOBVK0S2LC', 'yes'), ('A224TK7J4KA1LV', '2P3Z6R70G5R9R4QEF9574SSTVG2P9O', 'yes'), ('A224TK7J4KA1LV', '25UCF0CP5KXPC95DUBPGCPM7VGWWX9', 'yes'), ('A224TK7J4KA1LV', '2AOYTWX4H3H282M8LN7IEIJRJNW4ZT', 'no'), ('A224TK7J4KA1LV', '2J5JQ2172Z9998O6WZ3VMBU1B1FZR6', 'no'), ('A224TK7J4KA1LV', '2SIHFL42J1LYOZT2ELD0QGFKPVS2NE', 'yes'), ('A224TK7J4KA1LV', '2W36VCPWZ9RNWN0J7FJ70N3D1YKPA3', 'yes'), ('A224TK7J4KA1LV', '2F8S1CSTMOFXA4AZUQSE6DNXRXCROW', 'yes'), ('A224TK7J4KA1LV', '2TRKSSZVXX2QNWCPW4FYONIW95IN5Z', 'no'), ('A224TK7J4KA1LV', '2L13NAB6BFMFYFGTU7STRFDGMA2HU2', 'yes'), ('A224TK7J4KA1LV', '2B3RGNTBJ3MMWONA1LE6AKHBH439VT', 'yes'), ('A224TK7J4KA1LV', '24ZMVHVKCJ01KBK2GXY9SQW69QD758', 'yes'), ('A224TK7J4KA1LV', '2V99Y8TNKIMZBDNFRVJ2WM0POD59W3', 'yes'), ('A224TK7J4KA1LV', '2FK4E6AUBXHWB8PAXN0VJZWDZ3NX1B', 'yes'), ('A224TK7J4KA1LV', '2WAVGULF395SFX5BDTJ3VYNJJG5PBY', 'yes'), ('A224TK7J4KA1LV', '2DMZMAZHHRRLYHIJZ1MRP1KHWR4CLU', 'yes'), ('A224TK7J4KA1LV', '27E618N46UXFV4M09Q5TVDSN128RPD', 'yes'), ('A224TK7J4KA1LV', '2J9CJK63ZVE30IIF6W1QHJL5ZL59Y2', 'yes'), ('A224TK7J4KA1LV', '2DS6X3YOCCF0VGNF93KIVIIX6YURQ4', 'yes'), ('A224TK7J4KA1LV', '23ZGYQ4J3NABP2XHRTFY6IT1115PC5', 'yes'), ('A224TK7J4KA1LV', '2PFMB3Q39Z0BPLBJI7VE3STVZ26UJK', 'yes'), ('A224TK7J4KA1LV', '283FQ0ONNVRHKBZJLS7RJ76N3RBULM', 'no'), ('A224TK7J4KA1LV', '2F8S1CSTMOFXA4AZUQSE6DNXRXFORW', 'yes'), ('A224TK7J4KA1LV', '21UJ011K274JQ02L8KS8V46U5X5BD9', 'no'), ('A224TK7J4KA1LV', '2QAKMT6YTWX40UZX1PVDH9GIE0C0VB', 'yes'), ('A224TK7J4KA1LV', '2C18EBDPGFL6E37RBNINLWIO3JLH2M', 'yes'), ('A224TK7J4KA1LV', '2Y6VR14IXKO240KCQQNAEX3YWU59AV', 'yes'), ('A224TK7J4KA1LV', '2P7JWKUDD8XMUU8YLDRBEUTOEBXMB2', 'yes'), ('A224TK7J4KA1LV', '2525UFJ51Y7QXFHAA4J1KSTMWXQ9CX', 'yes'), ('A224TK7J4KA1LV', '2NCXFCD45XCECEVNZ7BIV3TGXMQWYF', 'yes'), ('A224TK7J4KA1LV', '27ZM8JIA0RYN20KIVYD0DBMJ1C8507', 'yes'), ('A224TK7J4KA1LV', '2I63W5XH0JPPB9KFASGL0P75S3FEKP', 'yes'), ('A224TK7J4KA1LV', '21RRJ85OZIZE0DS510FOY5EWXFB9XW', 'yes'), ('A224TK7J4KA1LV', '2LAMEZZ2MIKM60C31NP3GTWW50TN6F', 'yes'), ('A224TK7J4KA1LV', '241KM05BMJTU0PEXS7G9ZA7SDEFEJQ', 'yes'), ('A224TK7J4KA1LV', '2UZDM25L8I9NINOCQDXOM4QWFBYHRY', 'yes'), ('A224TK7J4KA1LV', '2J1E1M7PKK9LANOJBO30CLZXX3YZSU', 'yes'), ('A224TK7J4KA1LV', '2LWGDHDM25L8105U8K8E76OEC8MEO6', 'no'), ('A224TK7J4KA1LV', '2XANTL0XULRG6KTEF0DD55FPF5X2OT', 'yes'), ('A224TK7J4KA1LV', '2FC98JHSTLB34RX6VN9OJJEJZWORFD', 'yes'), ('A224TK7J4KA1LV', '2D2UDD8XMB3QM0HVNKLTW6T4MDIEPN', 'yes'), ('A224TK7J4KA1LV', '2C73TVOK5UFJOSG22SFZNYQS9UI743', 'yes'), ('A224TK7J4KA1LV', '25XXRDS4IC1EH45SVTD19A2I3GUWZ6', 'no'), ('A224TK7J4KA1LV', '2PG1THV8ER9YRK5FU0QSU5KFPAT2PA', 'yes'), ('A224TK7J4KA1LV', '2C79746OQ1SN9HPLILR59QKCV9RN39', 'yes'), ('A224TK7J4KA1LV', '2YDKCJ011K27NAP4W4N698N4ECQ9BQ', 'yes'), ('A224TK7J4KA1LV', '2XGQ4J3NAB6BYDXA0CPI11TJNV6ER,F', 'yes'), ('A224TK7J4KA1LV', '23K5L8I9NZW605H10SVQ47T8ER6UKJ', 'yes'), ('A224TK7J4KA1LV', '2ZCUU00OQKKAN90NK6TUA680MEQWWG', 'yes'), ('A224TK7J4KA1LV', '2P1HSTLB3L0FUARD0PAERREVMNHUI8', 'yes'), ('A224TK7J4KA1LV', '2581SNQQ7Q67JWJLWQER92TQKJN9TD', 'yes'), ('A224TK7J4KA1LV', '2LISPW1INMHGHHMEF11BEBFMNXHH4I', 'yes'), ('A224TK7J4KA1LV', '2SIHFL42J1LYOZT2ELD0QGFKPVRN2Y', 'yes'), ('A224TK7J4KA1LV', '2ZRNZW6HEZ6OXV8RJ7Z6HGUMAJRZPE', 'yes'), ('A224TK7J4KA1LV', '214T6YTWX4H30T76GR09OI6IR91X2S', 'yes'), ('A224TK7J4KA1LV', '2T5HMVHVKCJ0KS2XJIA7HKQWEJ1646', 'yes'), ('A224TK7J4KA1LV', '2ALDHJHP4BDD37MTK5JXHX3GBNY4XW', 'yes'), ('A224TK7J4KA1LV', '2IDP9746OQ1S6H822KY0D1QKK5I2ME', 'yes'), ('A224TK7J4KA1LV', '2WIOHOVR14IX3FKGLG8EKWA65LO76O', 'yes'), ('A224TK7J4KA1LV', '2GYKPEIB9BAW4J4S14WV5UD3UBHX08', 'yes'), ('A224TK7J4KA1LV', '2WQ06UFBNFSVZL3AFNWS46NG9XLH3X', 'yes'), ('A224TK7J4KA1LV', '2L13NAB6BFMFYFGTU7STRFDGMA2UHF', 'yes'), ('A224TK7J4KA1LV', '2ALDHJHP4BDD37MTK5JXHX3GBN04XY', 'no'), ('A224TK7J4KA1LV', '2F4NCWYB49F9BJAC9FSJWNSTDZYOSK', 'yes'), ('A224TK7J4KA1LV', '2BXBNFSVGULFM0NN8KEG9FS3VGE7LX', 'yes'), ('A224TK7J4KA1LV', '20JLY58B727MJ9YAWV41EYFS3XXU9S', 'no'), ('A224TK7J4KA1LV', '2FKAOO2GMMEG7FYL4QD6QAA20W877N', 'yes'), ('A224TK7J4KA1LV', '20JLY58B727MJ9YAWV41EYFS3XU9U4', 'yes'), ('A224TK7J4KA1LV', '2A79RA7S5WM1P0GN0TY0RMHLI0JZUA', 'yes'), ('A224TK7J4KA1LV', '2AOYTWX4H3H282M8LN7IEIJRJNY4ZV', 'yes'), ('A2ZUENR4ZLC3MN', '2K5AB6BFMFFOHP0OD7AFLGESKIHJW8', 'yes')] response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels)) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_estimated_workers_quality()) self.assertEqual('OK', response['status']) def test_GH_88(self): """ Test for github issue https://github.com/ipeirotis/Troia-Server/issues/88 Thanks to PeccatorImpius for reporting """ categories = ['blank', 'confirm_closed', 'confirm_no', 'confirm_nonrestaurant', 'confirm_yes'] algorithm = "BDS" # TODO XXX FIXME currently it won't work on IDS response = self.client.create(categories, prioritycalculator="CostBased", costMatrix=[ {"value":5.0, "to":"blank", "from":"blank"}, {"value":5.0, "to":"confirm_closed", "from":"blank"}, {"value":5.0, "to":"confirm_no", "from":"blank"}, {"value":5.0, "to":"confirm_nonrestaurant", "from":"blank"}, {"value":5.0, "to":"confirm_yes", "from":"blank"}, {"value":5.0, "to":"blank", "from":"confirm_closed"}, {"value":0.0, "to":"confirm_closed", "from":"confirm_closed"}, {"value":5.0, "to":"confirm_no", "from":"confirm_closed"}, {"value":5.0, "to":"confirm_nonrestaurant", "from":"confirm_closed"}, {"value":5.0, "to":"confirm_yes", "from":"confirm_closed"}, {"value":1.0, "to":"blank", "from":"confirm_no"}, {"value":1.0, "to":"confirm_closed", "from":"confirm_no"}, {"value":0.0, "to":"confirm_no", "from":"confirm_no"}, {"value":1.0, "to":"confirm_nonrestaurant", "from":"confirm_no"}, {"value":1.0, "to":"confirm_yes", "from":"confirm_no"}, {"value":5.0, "to":"blank", "from":"confirm_nonrestaurant"}, {"value":5.0, "to":"confirm_closed", "from":"confirm_nonrestaurant"}, {"value":5.0, "to":"confirm_no", "from":"confirm_nonrestaurant"}, {"value":0.0, "to":"confirm_nonrestaurant", "from":"confirm_nonrestaurant"}, {"value":5.0, "to":"confirm_yes", "from":"confirm_nonrestaurant"}, {"value":1.0, "to":"blank", "from":"confirm_yes"}, {"value":1.0, "to":"confirm_closed", "from":"confirm_yes"}, {"value":1.0, "to":"confirm_no", "from":"confirm_yes"}, {"value":1.0, "to":"confirm_nonrestaurant", "from":"confirm_yes"}, {"value":0.0, "to":"confirm_yes", "from":"confirm_yes"}], epsilon=0.0001, iterations=10, algorithm=algorithm, scheduler='NormalScheduler') self.assertEqual('OK', response['status']) assignedLabels = [ ('A1R7CJMWXC79UO', '2OVZKPFE4EGD044XHZZIHNZWGWV7HC', 'confirm_yes'), ('A2VRQML8Q3XYP4', '2OVZKPFE4EGD044XHZZIHNZWGWV7HC', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2OVZKPFE4EGD044XHZZIHNZWGWV7HC', 'confirm_yes'), ('A3TXO6RKFIDFUV', '2Z3KH1Q6SVQ80ZQH28XLDOBVMXHL2G', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2Z3KH1Q6SVQ80ZQH28XLDOBVMXHL2G', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2Z3KH1Q6SVQ80ZQH28XLDOBVMXHL2G', 'confirm_yes'), ('A1FQYUBCBNTQX7', '2V6ZFYQS1CST5FXS3RJ4QC1E8S3KN3', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2V6ZFYQS1CST5FXS3RJ4QC1E8S3KN3', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2V6ZFYQS1CST5FXS3RJ4QC1E8S3KN3', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2MCFJ51Y7QEOI6GL4F3S1MOF76TBEL', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2MCFJ51Y7QEOI6GL4F3S1MOF76TBEL', 'confirm_yes'), ('A3TXO6RKFIDFUV', '2MCFJ51Y7QEOI6GL4F3S1MOF76TBEL', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2S4JTUHYW2GT8095J6WWU169874PK4', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2S4JTUHYW2GT8095J6WWU169874PK4', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2S4JTUHYW2GT8095J6WWU169874PK4', 'confirm_no'), ('A3PRF4IIM2IQN2', '2S3YHVI44OS23CC3P15Z0BAPFQI4YO', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2S3YHVI44OS23CC3P15Z0BAPFQI4YO', 'confirm_closed'), ('AH5ZIMHL7TNWC', '2S3YHVI44OS23CC3P15Z0BAPFQI4YO', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2KXDEY5COW0LZIHYKQIFTC6VX9G4VF', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2KXDEY5COW0LZIHYKQIFTC6VX9G4VF', 'confirm_yes'), ('A2KET1HL1COET5', '2KXDEY5COW0LZIHYKQIFTC6VX9G4VF', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2FT49F9SSSHXKS1JZ6K5P5F8TJ4TX2', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2FT49F9SSSHXKS1JZ6K5P5F8TJ4TX2', 'confirm_no'), ('A3TXO6RKFIDFUV', '2FT49F9SSSHXKS1JZ6K5P5F8TJ4TX2', 'confirm_no'), ('A3PRF4IIM2IQN2', '27G2RCJK63ZVXUZMCYLIIQ9JVK87WC', 'confirm_yes'), ('A19BL9GZGXFWFT', '27G2RCJK63ZVXUZMCYLIIQ9JVK87WC', 'confirm_yes'), ('AH5ZIMHL7TNWC', '27G2RCJK63ZVXUZMCYLIIQ9JVK87WC', 'confirm_no'), ('AH5ZIMHL7TNWC', '2JODG67D1X3VJ62VO2B2RE1MH40HAC', 'confirm_no'), ('A31X3JCHS0BPFJ', '2JODG67D1X3VJ62VO2B2RE1MH40HAC', 'confirm_yes'), ('A2VRQML8Q3XYP4', '2JODG67D1X3VJ62VO2B2RE1MH40HAC', 'confirm_yes'), ('A31X3JCHS0BPFJ', '26ZIT3HVWAVK1XKIV4T1F2Z9J5XEMD', 'confirm_closed'), ('A3TXO6RKFIDFUV', '26ZIT3HVWAVK1XKIV4T1F2Z9J5XEMD', 'confirm_closed'), ('AH5ZIMHL7TNWC', '26ZIT3HVWAVK1XKIV4T1F2Z9J5XEMD', 'confirm_nonrestaurant'), ('A3PRF4IIM2IQN2', '20N1Y7QEOZFY9JJ747DONXRD2JYEH4', 'confirm_yes'), ('A31X3JCHS0BPFJ', '20N1Y7QEOZFY9JJ747DONXRD2JYEH4', 'confirm_yes'), ('A2KET1HL1COET5', '20N1Y7QEOZFY9JJ747DONXRD2JYEH4', 'confirm_yes'), ('A3PRF4IIM2IQN2', '241KM05BMJTU0PEXS7G9ZA7SFB2EJ9', 'confirm_yes'), ('A2L2ZXRHK3SNOP', '241KM05BMJTU0PEXS7G9ZA7SFB2EJ9', 'confirm_yes'), ('AH5ZIMHL7TNWC', '241KM05BMJTU0PEXS7G9ZA7SFB2EJ9', 'confirm_no'), ('A3PRF4IIM2IQN2', '2YDWAVKI62NJ9TJ2ED09YH6BU0SRJZ', 'confirm_yes'), ('A19BL9GZGXFWFT', '2YDWAVKI62NJ9TJ2ED09YH6BU0SRJZ', 'confirm_closed'), ('AH5ZIMHL7TNWC', '2YDWAVKI62NJ9TJ2ED09YH6BU0SRJZ', 'confirm_no'), ('A3PRF4IIM2IQN2', '2JV21O3W5XH02G7NUGBYMPLSZMLHB7', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2JV21O3W5XH02G7NUGBYMPLSZMLHB7', 'confirm_yes'), ('A2L2ZXRHK3SNOP', '2JV21O3W5XH02G7NUGBYMPLSZMLHB7', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2JS1QP6AUC26W7O2PFO330FKAR5811', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2JS1QP6AUC26W7O2PFO330FKAR5811', 'confirm_yes'), ('A1I3CXC17NIRWB', '2JS1QP6AUC26W7O2PFO330FKAR5811', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2CIFK0COK2JEKDPKWY0LZW6O9PSRKY', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2CIFK0COK2JEKDPKWY0LZW6O9PSRKY', 'confirm_yes'), ('A2KET1HL1COET5', '2CIFK0COK2JEKDPKWY0LZW6O9PSRKY', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2IJK274J79KQFXJ3ZIXU5FCDEKDHFP', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2IJK274J79KQFXJ3ZIXU5FCDEKDHFP', 'confirm_no'), ('A2L2ZXRHK3SNOP', '2IJK274J79KQFXJ3ZIXU5FCDEKDHFP', 'confirm_yes'), ('A3TXO6RKFIDFUV', '234EGOOGQSCMP9S5E65I2UU0A36EEO', 'confirm_closed'), ('A3PRF4IIM2IQN2', '234EGOOGQSCMP9S5E65I2UU0A36EEO', 'confirm_closed'), ('A31X3JCHS0BPFJ', '234EGOOGQSCMP9S5E65I2UU0A36EEO', 'confirm_closed'), ('A3TXO6RKFIDFUV', '23QJIA0RYNJ9LE1FYEWBUJTURDD27P', 'confirm_yes'), ('A3PRF4IIM2IQN2', '23QJIA0RYNJ9LE1FYEWBUJTURDD27P', 'confirm_yes'), ('A31X3JCHS0BPFJ', '23QJIA0RYNJ9LE1FYEWBUJTURDD27P', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2S5VP3TVOK5UYANWALHEWZFY07I52D', 'confirm_no'), ('A2L2ZXRHK3SNOP', '2S5VP3TVOK5UYANWALHEWZFY07I52D', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2S5VP3TVOK5UYANWALHEWZFY07I52D', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2ETAT2D5NQYNO6LLCX751YMNMBF84E', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2ETAT2D5NQYNO6LLCX751YMNMBF84E', 'confirm_no'), ('A2L2ZXRHK3SNOP', '2ETAT2D5NQYNO6LLCX751YMNMBF84E', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2Z2MN9U8P9Y3RKER9WUS2YIMAY9EVU', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2Z2MN9U8P9Y3RKER9WUS2YIMAY9EVU', 'confirm_no'), ('A1I3CXC17NIRWB', '2Z2MN9U8P9Y3RKER9WUS2YIMAY9EVU', 'confirm_yes'), ('ABT7QTMIYXYO0', '2GXYQS1CSTMOYO984I9C9EYDXCZPM9', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2GXYQS1CSTMOYO984I9C9EYDXCZPM9', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2GXYQS1CSTMOYO984I9C9EYDXCZPM9', 'confirm_no'), ('AH5ZIMHL7TNWC', '2BLK4F0OHOVRKV0SW2TLH2HEMBR23K', 'blank'), ('A3PRF4IIM2IQN2', '2BLK4F0OHOVRKV0SW2TLH2HEMBR23K', 'confirm_yes'), ('A2L2ZXRHK3SNOP', '2BLK4F0OHOVRKV0SW2TLH2HEMBR23K', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2X6OGQSCM6IATTA9U8LU80OQUZQHHN', 'confirm_yes'), ('A2KET1HL1COET5', '2X6OGQSCM6IATTA9U8LU80OQUZQHHN', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2X6OGQSCM6IATTA9U8LU80OQUZQHHN', 'confirm_yes'), ('A31X3JCHS0BPFJ', '29UWY2AOO2GM55YJ0UHSKM6IKPI442', 'confirm_yes'), ('A3PRF4IIM2IQN2', '29UWY2AOO2GM55YJ0UHSKM6IKPI442', 'confirm_yes'), ('AH5ZIMHL7TNWC', '29UWY2AOO2GM55YJ0UHSKM6IKPI442', 'confirm_yes'), ('A31X3JCHS0BPFJ', '2DJVP9746OQ1BE8LJ4X7851QUR3L1Z', 'confirm_yes'), ('ABT7QTMIYXYO0', '2DJVP9746OQ1BE8LJ4X7851QUR3L1Z', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2DJVP9746OQ1BE8LJ4X7851QUR3L1Z', 'confirm_yes'), ('A31X3JCHS0BPFJ', '25XXRDS4IC1EH45SVTD19A2I5DKWZS', 'confirm_yes'), ('A3PRF4IIM2IQN2', '25XXRDS4IC1EH45SVTD19A2I5DKWZS', 'confirm_no'), ('AH5ZIMHL7TNWC', '25XXRDS4IC1EH45SVTD19A2I5DKWZS', 'confirm_no'), ('A2VRQML8Q3XYP4', '2DABRI8IUB2YD0QET6KLJ3L0PQZH56', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2DABRI8IUB2YD0QET6KLJ3L0PQZH56', 'confirm_no'), ('A3PRF4IIM2IQN2', '2DABRI8IUB2YD0QET6KLJ3L0PQZH56', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2TOSK9RJ85OZ1QWCYO1PUOOQFTCU60', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2TOSK9RJ85OZ1QWCYO1PUOOQFTCU60', 'confirm_yes'), ('A2L2ZXRHK3SNOP', '2TOSK9RJ85OZ1QWCYO1PUOOQFTCU60', 'confirm_yes'), ('A3PRF4IIM2IQN2', '2IBHV8ER9Y8T6B0HB6D5SFHSC334RM', 'confirm_yes'), ('ABT7QTMIYXYO0', '2IBHV8ER9Y8T6B0HB6D5SFHSC334RM', 'confirm_yes'), ('AH5ZIMHL7TNWC', '2IBHV8ER9Y8T6B0HB6D5SFHSC334RM', 'confirm_no'), ('A3TXO6RKFIDFUV', '294EZZ2MIKMNSLQKLCU81WWXSI97O0', 'confirm_yes'), ('A19BL9GZGXFWFT', '294EZZ2MIKMNSLQKLCU81WWXSI97O0', 'confirm_yes'), ('A1I3CXC17NIRWB', '294EZZ2MIKMNSLQKLCU81WWXSI97O0', 'confirm_yes'), ('A2VRQML8Q3XYP4', '2AOYTWX4H3H282M8LN7IEIJRLKM4ZF', 'confirm_yes')] response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels)) self.assertEqual('OK', response['status']) if algorithm == "BDS": self.client.await_completion(self.client.post_compute()) response = self.client.await_completion(self.client.get_estimated_workers_quality()) self.assertEqual('OK', response['status']) for w in response['result']: val = float(w['value']) self.assertFalse(math.isnan(val))