def createNominalJob(): client = TroiaClient(ADDRESS) client.create(CATEGORIES) jobs.append((client.jid, "NOMINAL")) check_status(client, client.post_assigned_labels(ASSIGNED_LABELS)) check_status(client, client.post_evaluation_objects(EVALUATION_DATA)) check_status(client, client.post_compute()) check_assigns(client, len(ASSIGNED_LABELS))
class TestEvaluationLabels(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) response = self.client.create(CATEGORIES) self.assertEqual('OK', response['status']) def tearDown(self): self.client.delete() def _test_method(self, eval_data): response = self.client.await_completion(self.client.post_evaluation_objects(eval_data)) self.assertEqual('OK', response['status']) #get the unassigned labels response = self.client.await_completion(self.client.get_evaluation_objects()) self.assertEqual('OK', response['status']) result = response['result'] self.assertEqual(len(eval_data), len(result)) results = [(evaluationLabel['name'], evaluationLabel['evaluationLabel']) for evaluationLabel in response['result']] for evalLabel in eval_data: self.assertTrue(evalLabel in results) def test_AddGetEmptyEvaluationLabels(self): self._test_method([]) def test_AddGetEvaluationLabels_PrintableASCII_RegularChars(self): self._test_method(EVALUATION_DATA) def test_AddGetEvaluationLabel_PrintableASCII_SpecialChars(self): self._test_method([('~!@%#^&*()_+=-[]{}:<>,./', 'notporn')]) def test_AddGetEvaluationLabel_ExtendedASCIIChars(self): self._test_method([(u'™ž¤©', 'notporn')]) def test_AddGetEvaluationLabel_UnicodeChars(self): self._test_method([(u'ૉେஇ', 'notporn')]) def test_AddGetEvaluationLabel_LongLabelNames(self): response = self.client.await_completion(self.client.post_evaluation_objects([('sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs', 'notporn')])) self.assertEqual('ERROR', response['status']) self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result'])
class TestPrediction(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) self.client.create(CATEGORIES, costMatrix=COST_MATRIX, categoryPriors=CATEGORY_PRIORS, algorithm=self.algorithm) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_evaluation_objects(EVALUATION_DATA)) self.client.await_completion(self.client.post_compute()) def tearDown(self): self.client.delete() def _getObjectsPrediction(self, method, expectedResults): response = self.client.await_completion(self.client.get_objects_prediction(method)) self.assertEqual('OK', response['status']) for categories in response['result']: self.assertEqual(expectedResults[categories['objectName']], categories['categoryName']) def _getEstimatedObjectsCost(self, costMethod, expectedCosts): response = self.client.await_completion(self.client.get_estimated_objects_cost(costMethod)) self.assertEqual('OK', response['status']) for cost in response['result']: self.assertTrue(abs(expectedCosts[cost['objectName']] - cost['value']) / 100 < TOLERANCE) def _getEvaluatedObjectsCost(self, labelChoosingMethod, expectedResults): response = self.client.await_completion(self.client.get_evaluated_objects_cost(labelChoosingMethod)) self.assertEqual('OK', response['status']) for item in response['result']: self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE) def _getEstimatedObjectsQuality(self, costAlgorithm, expectedDataQuality): response = self.client.await_completion(self.client.get_estimated_objects_quality(costAlgorithm)) self.assertEqual('OK', response['status']) for dataQuality in response['result']: self.assertTrue(abs(expectedDataQuality[dataQuality['objectName']] - dataQuality['value']) / 100 < TOLERANCE) def _getEvaluatedObjectsQuality(self, labelChoosingMethod, expectedResults): response = self.client.await_completion(self.client.get_evaluated_objects_quality(labelChoosingMethod)) self.assertEqual('OK', response['status']) for item in response['result']: self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE) def _getEstimatedObjectsCostSummary (self, expectedObjectsCosts): response = self.client.await_completion(self.client.get_objects_cost_estimated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedObjectsCosts.items(): self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE) def _getEvaluatedObjectsCostSummary (self, expectedObjectsCosts): response = self.client.await_completion(self.client.get_objects_cost_evaluated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedObjectsCosts.items(): self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE) def _getEstimatedObjectsQualitySummary(self, expectedDataQuality): response = self.client.await_completion(self.client.get_objects_quality_estimated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedDataQuality.items(): self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE) def _getEvaluatedObjectsQualitySummary(self, expectedDataQuality): response = self.client.await_completion(self.client.get_objects_quality_evaluated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedDataQuality.items(): self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE) def _getEstimatedWorkerCost(self, costAlgorithm, expectedWorkerCost): response = self.client.await_completion(self.client.get_estimated_workers_cost(costAlgorithm)) self.assertEqual('OK', response['status']) for workerCost in response['result']: self.assertTrue(abs(expectedWorkerCost[workerCost['workerName']] - workerCost['value']) / 100 < TOLERANCE) def _getEstimatedWorkerQuality(self, costAlgorithm, expectedWorkerQuality): response = self.client.await_completion(self.client.get_estimated_workers_quality(costAlgorithm)) self.assertEqual('OK', response['status']) for workerQuality in response['result']: self.assertTrue(abs(expectedWorkerQuality[workerQuality['workerName']] - workerQuality['value']) / 100 < TOLERANCE) def _getEvaluatedWorkerQuality(self, costAlgorithm, expectedResults): response = self.client.await_completion(self.client.get_evaluated_workers_quality(costAlgorithm)) self.assertEqual('OK', response['status']) for item in response['result']: self.assertTrue(abs(expectedResults[item['workerName']] - item['value']) / 100 < TOLERANCE) def _getWorkersQualityEstimatedSummary(self, expectedWorkerQuality): response = self.client.await_completion(self.client.get_workers_quality_estimated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedWorkerQuality.items(): self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE) def _getWorkersQualityEvaluatedSummary(self, expectedWorkerQuality): response = self.client.await_completion(self.client.get_workers_quality_evaluated_summary()) self.assertEqual('OK', response['status']) for k, v in expectedWorkerQuality.items(): self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE) def _getCategoryProbability(self, expectedProbabilities): for object in set((x[1] for x in ASSIGNED_LABELS)): response = self.client.await_completion(self.client.get_probability_distribution(object)) self.assertEqual('OK', response['status']) self.assertEqual(expectedProbabilities[object], response['result'])
print algorithm client = TroiaClient(ADDRESS) categories = dataProcessor.time_data_load_call(lambda: dataProcessor.loadCategories(categoriesFile)) goldLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadGoldLabels(goldLabelsFile)) evaluationLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadEvaluationLabels(evaluationLabelsFile)) assignedLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadAssigns(assignsFile)) print 'Creating client' createClient_t = dataProcessor.time_api_call(lambda: client.create(categories, algorithm=algorithm)) print 'Post gold labels' loadGolds_t = dataProcessor.time_api_call(lambda: client.post_gold_data(goldLabels)) print 'Post evaluation labels' loadEval_t = dataProcessor.time_api_call(lambda: client.post_evaluation_objects(evaluationLabels)) print 'Post assigned labels' loadAssigns_t = dataProcessor.post_assigns(client, assignedLabels) print 'Computing' compute_t = dataProcessor.time_api_call(lambda: client.post_compute()) print 'Getting object prediction' oPred_t = dataProcessor.time_api_call(lambda: client.get_objects_prediction("MinCost")) print 'Getting worker quality summary' wqEst_t = dataProcessor.time_api_call(lambda: client.get_workers_quality_estimated_summary()) results_writer.writerow([dataSet, algorithm, createClient_t, loadAssigns_t, loadGolds_t, loadEval_t, compute_t, oPred_t, wqEst_t])