class TestPrediction(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
self.client.create(CATEGORIES, costMatrix=COST_MATRIX, categoryPriors=CATEGORY_PRIORS, algorithm=self.algorithm)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_evaluation_objects(EVALUATION_DATA))
self.client.await_completion(self.client.post_compute())
def tearDown(self):
self.client.delete()
def _getObjectsPrediction(self, method, expectedResults):
response = self.client.await_completion(self.client.get_objects_prediction(method))
self.assertEqual('OK', response['status'])
for categories in response['result']:
self.assertEqual(expectedResults[categories['objectName']], categories['categoryName'])
def _getEstimatedObjectsCost(self, costMethod, expectedCosts):
response = self.client.await_completion(self.client.get_estimated_objects_cost(costMethod))
self.assertEqual('OK', response['status'])
for cost in response['result']:
self.assertTrue(abs(expectedCosts[cost['objectName']] - cost['value']) / 100 < TOLERANCE)
def _getEvaluatedObjectsCost(self, labelChoosingMethod, expectedResults):
response = self.client.await_completion(self.client.get_evaluated_objects_cost(labelChoosingMethod))
self.assertEqual('OK', response['status'])
for item in response['result']:
self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE)
def _getEstimatedObjectsQuality(self, costAlgorithm, expectedDataQuality):
response = self.client.await_completion(self.client.get_estimated_objects_quality(costAlgorithm))
self.assertEqual('OK', response['status'])
for dataQuality in response['result']:
self.assertTrue(abs(expectedDataQuality[dataQuality['objectName']] - dataQuality['value']) / 100 < TOLERANCE)
def _getEvaluatedObjectsQuality(self, labelChoosingMethod, expectedResults):
response = self.client.await_completion(self.client.get_evaluated_objects_quality(labelChoosingMethod))
self.assertEqual('OK', response['status'])
for item in response['result']:
self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE)
def _getEstimatedObjectsCostSummary (self, expectedObjectsCosts):
response = self.client.await_completion(self.client.get_objects_cost_estimated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedObjectsCosts.items():
self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEvaluatedObjectsCostSummary (self, expectedObjectsCosts):
response = self.client.await_completion(self.client.get_objects_cost_evaluated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedObjectsCosts.items():
self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEstimatedObjectsQualitySummary(self, expectedDataQuality):
response = self.client.await_completion(self.client.get_objects_quality_estimated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedDataQuality.items():
self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEvaluatedObjectsQualitySummary(self, expectedDataQuality):
response = self.client.await_completion(self.client.get_objects_quality_evaluated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedDataQuality.items():
self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEstimatedWorkerCost(self, costAlgorithm, expectedWorkerCost):
response = self.client.await_completion(self.client.get_estimated_workers_cost(costAlgorithm))
self.assertEqual('OK', response['status'])
for workerCost in response['result']:
self.assertTrue(abs(expectedWorkerCost[workerCost['workerName']] - workerCost['value']) / 100 < TOLERANCE)
def _getEstimatedWorkerQuality(self, costAlgorithm, expectedWorkerQuality):
response = self.client.await_completion(self.client.get_estimated_workers_quality(costAlgorithm))
self.assertEqual('OK', response['status'])
for workerQuality in response['result']:
self.assertTrue(abs(expectedWorkerQuality[workerQuality['workerName']] - workerQuality['value']) / 100 < TOLERANCE)
def _getEvaluatedWorkerQuality(self, costAlgorithm, expectedResults):
response = self.client.await_completion(self.client.get_evaluated_workers_quality(costAlgorithm))
self.assertEqual('OK', response['status'])
for item in response['result']:
self.assertTrue(abs(expectedResults[item['workerName']] - item['value']) / 100 < TOLERANCE)
def _getWorkersQualityEstimatedSummary(self, expectedWorkerQuality):
response = self.client.await_completion(self.client.get_workers_quality_estimated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedWorkerQuality.items():
self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getWorkersQualityEvaluatedSummary(self, expectedWorkerQuality):
response = self.client.await_completion(self.client.get_workers_quality_evaluated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedWorkerQuality.items():
self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getCategoryProbability(self, expectedProbabilities):
for object in set((x[1] for x in ASSIGNED_LABELS)):
response = self.client.await_completion(self.client.get_probability_distribution(object))
self.assertEqual('OK', response['status'])
self.assertEqual(expectedProbabilities[object], response['result'])
class TestUnassignedLabels(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def get_priors(self, categories):
return [{'categoryName': c, "value": 1. / len(categories)} for c in categories]
def tearDown(self):
self.client.delete()
def _test_method(self, categories, priors, unassignedLabels, expectedProbabilities):
response = self.client.create(categories, categoryPriors=priors, algorithm="BMV")
self.assertEqual('OK', response['status'])
#post the unassigned labels
response = self.client.await_completion(self.client.post_objects(unassignedLabels))
self.assertEqual('OK', response['status'])
#get the unassigned labels
response = self.client.await_completion(self.client.get_objects())
self.assertEqual('OK', response['status'])
self.assertEqual(len(unassignedLabels), len(response['result']))
if (unassignedLabels):
result = response['result'][0]
self.assertEqual(unassignedLabels[0], result['name'])
def test_AddGetObjects_EmptyLabels(self):
self._test_method(CATEGORIES, CATEGORY_PRIORS, [], [])
def test_AddGetUnassignedLabels_LongLabelNames(self):
categories = ["category1", "category2"]
priors = [{"categoryName": "category1", "value": 0.0000000001}, {"categoryName": "category2", "value": 0.9999999999}]
unassignedLabels = ["sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs"]
expectedProbabilities = [('category1', 0.5), ('category2', 0.5)]
response = self.client.create(categories, categoryPriors=priors, algorithm="BMV")
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_objects(unassignedLabels))
self.assertEqual('ERROR', response['status'])
self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result'])
def test_AddGetUnassignedLabels_PrintableASCII_RegularChars(self):
categories = ["category1", "category2", "category3", "category4"]
unassignedLabels = ["testObject1"]
expectedProbabilities = [('category1', 0.25),
('category2', 0.25),
('category3', 0.25),
('category4', 0.25)]
self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)
def test_AddGetUnassignedLabels_PrintableASCII_SpecialChars(self):
categories = ["category1", "category2", "category3"]
unassignedLabels = ["~!@#%^&*()_+=-[]{}:<>,./"]
expectedProbabilities = [('category1', 0.3333333333333333),
('category2', 0.3333333333333333),
('category3', 0.3333333333333333)]
self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)
def test_AddGetUnassignedLabels_ExtendedASCIIChars(self):
categories = ["category1", "category2"]
unassignedLabels = [u"ëñµ¼Úæ"]
expectedProbabilities = [('category1', 0.5), ('category2', 0.5)]
self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)
def test_AddGetUnassignedLabels_UnicodeChars(self):
categories = ["category1", "category2"]
unassignedLabels = [u"ూഹܬआਖ਼"]
expectedProbabilities = [('category1', 0.5), ('category2', 0.5)]
self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)
def test_GetObjectAssigns(self):
response = self.client.create(CATEGORIES)
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.assertEqual('OK', response['status'])
#post the unassigned label
unassignedLabel = [u"ూഹܬआਖ਼"]
response = self.client.await_completion(self.client.post_objects(unassignedLabel))
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.get_objects())
self.assertEqual('OK', response['status'])
result = response['result']
self.assertEqual(6, len(result))
results = []
for object in result:
if object['name'] not in unassignedLabel:
assigns = self.client.await_completion(self.client.get_object_assigns(object['name']))['result']
for a in assigns:
results.append((a['worker'], a['object'], a['label']))
for label in ASSIGNED_LABELS:
self.assertTrue(label in results)
def test_GetObjectInfo(self):
response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS)
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels([('worker1', 'url1', 'p**n'), ('worker1', 'url2', 'p**n')]))
self.assertEqual('OK', response['status'])
#post the unassigned label
objects_without_assigns = ["newUnassignedLabel"]
response = self.client.await_completion(self.client.post_objects(objects_without_assigns))
self.assertEqual('OK', response['status'])
#get labels
response = self.client.await_completion(self.client.get_objects())
self.assertEqual('OK', response['status'])
result = response['result']
self.assertEqual(3, len(result))
self.client.await_completion(self.client.post_compute())
for label in result:
response = self.client.await_completion(self.client.get_probability_distribution(label['name']))
dist = response['result'][0]
self.assertAlmostEqual(dist['value'], 0.5 if label['name'] in objects_without_assigns else 0.9 if dist['categoryName'] == 'p**n' else 0.1)