def createNominalJob():
client = TroiaClient(ADDRESS)
client.create(CATEGORIES)
jobs.append((client.jid, "NOMINAL"))
check_status(client, client.post_assigned_labels(ASSIGNED_LABELS))
check_status(client, client.post_evaluation_objects(EVALUATION_DATA))
check_status(client, client.post_compute())
check_assigns(client, len(ASSIGNED_LABELS))
class TestEvaluationLabels(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
response = self.client.create(CATEGORIES)
self.assertEqual('OK', response['status'])
def tearDown(self):
self.client.delete()
def _test_method(self, eval_data):
response = self.client.await_completion(self.client.post_evaluation_objects(eval_data))
self.assertEqual('OK', response['status'])
#get the unassigned labels
response = self.client.await_completion(self.client.get_evaluation_objects())
self.assertEqual('OK', response['status'])
result = response['result']
self.assertEqual(len(eval_data), len(result))
results = [(evaluationLabel['name'], evaluationLabel['evaluationLabel']) for evaluationLabel in response['result']]
for evalLabel in eval_data:
self.assertTrue(evalLabel in results)
def test_AddGetEmptyEvaluationLabels(self):
self._test_method([])
def test_AddGetEvaluationLabels_PrintableASCII_RegularChars(self):
self._test_method(EVALUATION_DATA)
def test_AddGetEvaluationLabel_PrintableASCII_SpecialChars(self):
self._test_method([('~!@%#^&*()_+=-[]{}:<>,./', 'notporn')])
def test_AddGetEvaluationLabel_ExtendedASCIIChars(self):
self._test_method([(u'™ž¤©', 'notporn')])
def test_AddGetEvaluationLabel_UnicodeChars(self):
self._test_method([(u'ૉେஇ', 'notporn')])
def test_AddGetEvaluationLabel_LongLabelNames(self):
response = self.client.await_completion(self.client.post_evaluation_objects([('sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs', 'notporn')]))
self.assertEqual('ERROR', response['status'])
self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result'])
class TestPrediction(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
self.client.create(CATEGORIES, costMatrix=COST_MATRIX, categoryPriors=CATEGORY_PRIORS, algorithm=self.algorithm)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_evaluation_objects(EVALUATION_DATA))
self.client.await_completion(self.client.post_compute())
def tearDown(self):
self.client.delete()
def _getObjectsPrediction(self, method, expectedResults):
response = self.client.await_completion(self.client.get_objects_prediction(method))
self.assertEqual('OK', response['status'])
for categories in response['result']:
self.assertEqual(expectedResults[categories['objectName']], categories['categoryName'])
def _getEstimatedObjectsCost(self, costMethod, expectedCosts):
response = self.client.await_completion(self.client.get_estimated_objects_cost(costMethod))
self.assertEqual('OK', response['status'])
for cost in response['result']:
self.assertTrue(abs(expectedCosts[cost['objectName']] - cost['value']) / 100 < TOLERANCE)
def _getEvaluatedObjectsCost(self, labelChoosingMethod, expectedResults):
response = self.client.await_completion(self.client.get_evaluated_objects_cost(labelChoosingMethod))
self.assertEqual('OK', response['status'])
for item in response['result']:
self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE)
def _getEstimatedObjectsQuality(self, costAlgorithm, expectedDataQuality):
response = self.client.await_completion(self.client.get_estimated_objects_quality(costAlgorithm))
self.assertEqual('OK', response['status'])
for dataQuality in response['result']:
self.assertTrue(abs(expectedDataQuality[dataQuality['objectName']] - dataQuality['value']) / 100 < TOLERANCE)
def _getEvaluatedObjectsQuality(self, labelChoosingMethod, expectedResults):
response = self.client.await_completion(self.client.get_evaluated_objects_quality(labelChoosingMethod))
self.assertEqual('OK', response['status'])
for item in response['result']:
self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE)
def _getEstimatedObjectsCostSummary (self, expectedObjectsCosts):
response = self.client.await_completion(self.client.get_objects_cost_estimated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedObjectsCosts.items():
self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEvaluatedObjectsCostSummary (self, expectedObjectsCosts):
response = self.client.await_completion(self.client.get_objects_cost_evaluated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedObjectsCosts.items():
self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEstimatedObjectsQualitySummary(self, expectedDataQuality):
response = self.client.await_completion(self.client.get_objects_quality_estimated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedDataQuality.items():
self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEvaluatedObjectsQualitySummary(self, expectedDataQuality):
response = self.client.await_completion(self.client.get_objects_quality_evaluated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedDataQuality.items():
self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEstimatedWorkerCost(self, costAlgorithm, expectedWorkerCost):
response = self.client.await_completion(self.client.get_estimated_workers_cost(costAlgorithm))
self.assertEqual('OK', response['status'])
for workerCost in response['result']:
self.assertTrue(abs(expectedWorkerCost[workerCost['workerName']] - workerCost['value']) / 100 < TOLERANCE)
def _getEstimatedWorkerQuality(self, costAlgorithm, expectedWorkerQuality):
response = self.client.await_completion(self.client.get_estimated_workers_quality(costAlgorithm))
self.assertEqual('OK', response['status'])
for workerQuality in response['result']:
self.assertTrue(abs(expectedWorkerQuality[workerQuality['workerName']] - workerQuality['value']) / 100 < TOLERANCE)
def _getEvaluatedWorkerQuality(self, costAlgorithm, expectedResults):
response = self.client.await_completion(self.client.get_evaluated_workers_quality(costAlgorithm))
self.assertEqual('OK', response['status'])
for item in response['result']:
self.assertTrue(abs(expectedResults[item['workerName']] - item['value']) / 100 < TOLERANCE)
def _getWorkersQualityEstimatedSummary(self, expectedWorkerQuality):
response = self.client.await_completion(self.client.get_workers_quality_estimated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedWorkerQuality.items():
self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getWorkersQualityEvaluatedSummary(self, expectedWorkerQuality):
response = self.client.await_completion(self.client.get_workers_quality_evaluated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedWorkerQuality.items():
self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getCategoryProbability(self, expectedProbabilities):
for object in set((x[1] for x in ASSIGNED_LABELS)):
response = self.client.await_completion(self.client.get_probability_distribution(object))
self.assertEqual('OK', response['status'])
self.assertEqual(expectedProbabilities[object], response['result'])
print algorithm
client = TroiaClient(ADDRESS)
categories = dataProcessor.time_data_load_call(lambda: dataProcessor.loadCategories(categoriesFile))
goldLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadGoldLabels(goldLabelsFile))
evaluationLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadEvaluationLabels(evaluationLabelsFile))
assignedLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadAssigns(assignsFile))
print 'Creating client'
createClient_t = dataProcessor.time_api_call(lambda: client.create(categories, algorithm=algorithm))
print 'Post gold labels'
loadGolds_t = dataProcessor.time_api_call(lambda: client.post_gold_data(goldLabels))
print 'Post evaluation labels'
loadEval_t = dataProcessor.time_api_call(lambda: client.post_evaluation_objects(evaluationLabels))
print 'Post assigned labels'
loadAssigns_t = dataProcessor.post_assigns(client, assignedLabels)
print 'Computing'
compute_t = dataProcessor.time_api_call(lambda: client.post_compute())
print 'Getting object prediction'
oPred_t = dataProcessor.time_api_call(lambda: client.get_objects_prediction("MinCost"))
print 'Getting worker quality summary'
wqEst_t = dataProcessor.time_api_call(lambda: client.get_workers_quality_estimated_summary())
results_writer.writerow([dataSet, algorithm, createClient_t, loadAssigns_t, loadGolds_t, loadEval_t, compute_t, oPred_t, wqEst_t])
