class TestGoldLabels(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) response = self.client.create(CATEGORIES) self.assertEqual("OK", response["status"]) def tearDown(self): self.client.delete() def _test_method(self, goldLabels): response = self.client.await_completion(self.client.post_gold_data(goldLabels)) self.assertEqual("OK", response["status"]) # get the gold labels response = self.client.await_completion(self.client.get_gold_data()) self.assertEqual("OK", response["status"]) result = response["result"] self.assertEqual(len(goldLabels), len(result)) results = [tuple(receivedLabel.values()) for receivedLabel in response["result"]] for label in goldLabels: self.assertTrue(label in results) def test_AddGetEmptyGoldLabels(self): self._test_method([]) def test_AddGetGoldLabel_LongLabelName(self): response = self.client.await_completion( self.client.post_gold_data( [ ( "sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs", "notporn", ) ] ) ) self.assertEqual("ERROR", response["status"]) self.assertEqual("Internal error: Object name should be shorter than 100 chars", response["result"]) def test_AddGetGoldLabel_PrintableASCII_RegularChars(self): self._test_method([("url1", "notporn")]) def test_AddGetGoldLabel_PrintableASCII_SpecialChars(self): self._test_method([("~!@#^&*()_+=-[]{}:<>,./", "notporn")]) def test_AddGetGoldLabel_ExtendedASCIIChars(self): self._test_method([(u"™ž¤©", "notporn")]) def test_AddGetGoldLabel_UnicodeChars(self): self._test_method([(u"ૉେஇ", "notporn")])
class NominalSimulation(Simulation): def __init__(self, algorithm, *args, **kwargs): super(NominalSimulation, self).__init__(*args, **kwargs) self.algorithm = algorithm def create(self): self.tc = TroiaClient(TROIA_ADDRESS) categories, init_kwargs = self.data_generator.gen_init_data() self.tc.create(categories, algorithm=self.algorithm, **init_kwargs) def _upload_golds(self, golds): return ret_exectime(self.tc, self.tc.post_gold_data(golds)) def _upload_assigns_package(self, assigns): return ret_exectime(self.tc, self.tc.post_assigned_labels(assigns)) def compute(self): return ret_exectime(self.tc, self.tc.post_compute()) def shutdown(self): self.tc.delete()
class TestWorkers(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) def tearDown(self): self.client.delete() def test_AddGetEmptyWorkers(self): self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS) self.client.await_completion(self.client.post_compute()) response = self.client.await_completion(self.client.get_workers()) self.assertEqual('OK', response['status']) self.assertEqual([], response['result']) def test_AddGetWorkers_BeforeCompute(self): self.client.create(CATEGORIES) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES)) response = self.client.await_completion(self.client.get_workers()) self.assertEqual(5, len(response['result'])) for w in response['result']: self.assertEqual(5, w['value']['assigns']) def test_AddGetWorkers_AfterCompute(self): self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_compute()) #assigns check response = self.client.await_completion(self.client.get_workers()) self.assertEqual(5, len(response['result'])) for w in response['result']: self.assertEqual(5, w['value']['assigns']) #confusion matrices check response = self.client.await_completion(self.client.get_workers_confusion_matrix()) exp = { u'worker1': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, {u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}], u'worker3': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, {u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}], u'worker2': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.}, {u'to': u'p**n', u'from': u'notporn', u'value': 0.3333}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.6666}], u'worker5': [{u'to': u'p**n', u'from': u'p**n', u'value': 0.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 1.0}, {u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}], u'worker4': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0}, {u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}]} for w in response['result']: worker_name = w['workerName'] for e1 in w['value']: exists = False for e2 in exp[worker_name]: if e2['to'] == e1['to'] and e2['from'] == e1['from']: exists = True self.assertAlmostEqual( e2['value'], e1['value'], 2, "{}, from: {}, to: {}. expected:{}, was: {}".format(worker_name, e1['from'], e1['to'], e2['value'], e1['value'])) self.assertTrue(exists) def test_GetWorkerInfo(self): self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES)) self.client.await_completion(self.client.post_compute()) response = self.client.await_completion(self.client.get_worker_info('worker1')) self.assertTrue(5, response['result']['value']['assigns']) self.assertEqual(2, response['result']['value']['goldTests']) self.assertEqual(1, response['result']['value']['correctGoldTests']) self.assertTrue('worker1', response['result']['workerName']) def test_GetWorkerAssigns(self): self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX) self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS)) self.client.await_completion(self.client.post_compute()) response = self.client.await_completion(self.client.get_worker_assigns('worker1')) print response self.assertTrue(5, len(response['result'])) for a in response['result']: self.assertTrue((a['worker'], a['object'], a['label']) in ASSIGNED_LABELS)
evaluationLabelsFile = os.path.join(testFolder, 'evaluation.txt') for algorithm in algorithms: print '--------' print algorithm client = TroiaClient(ADDRESS) categories = dataProcessor.time_data_load_call(lambda: dataProcessor.loadCategories(categoriesFile)) goldLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadGoldLabels(goldLabelsFile)) evaluationLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadEvaluationLabels(evaluationLabelsFile)) assignedLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadAssigns(assignsFile)) print 'Creating client' createClient_t = dataProcessor.time_api_call(lambda: client.create(categories, algorithm=algorithm)) print 'Post gold labels' loadGolds_t = dataProcessor.time_api_call(lambda: client.post_gold_data(goldLabels)) print 'Post evaluation labels' loadEval_t = dataProcessor.time_api_call(lambda: client.post_evaluation_objects(evaluationLabels)) print 'Post assigned labels' loadAssigns_t = dataProcessor.post_assigns(client, assignedLabels) print 'Computing' compute_t = dataProcessor.time_api_call(lambda: client.post_compute()) print 'Getting object prediction' oPred_t = dataProcessor.time_api_call(lambda: client.get_objects_prediction("MinCost")) print 'Getting worker quality summary' wqEst_t = dataProcessor.time_api_call(lambda: client.get_workers_quality_estimated_summary())
class TestQualitySensitivePayments(unittest.TestCase): def setUp(self): self.client = TroiaClient(ADDRESS) def tearDown(self): self.client.delete() def testQSP_ZeroWorkerQUality(self): assignedLabels = [ ('worker1', 'url1', 'cat2'), ('worker1', 'url2', 'cat1'), ('worker2', 'url1', 'cat1'), ('worker2', 'url2', 'cat1'), ('worker3', 'url1', 'cat1'), ('worker3', 'url2', 'cat2')] goldLabels = [ ('url1', 'cat1'), ('url2', 'cat2') ] response = self.client.create(["cat1", "cat2"]) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels)) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_gold_data(goldLabels)) self.assertEqual('OK', response['status']) response = self.client.post_compute() self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost")) self.assertEqual('OK', response['status']) pprint(response) response = self.client.await_completion(self.client.get_workers_quality_payment()) pprint(response) def testQSP_NegativeWorkerQuality(self): assignedLabels = [ ('worker1', 'url1', 'cat2'), ('worker1', 'url2', 'cat1'), ('worker1', 'url3', 'cat2'), ('worker2', 'url1', 'cat1'), ('worker2', 'url2', 'cat1'), ('worker2', 'url3', 'cat1'), ('worker3', 'url1', 'cat1'), ('worker3', 'url2', 'cat2'), ('worker3', 'url3', 'cat1') ] goldLabels = [ ('url1', 'cat1'), ('url2', 'cat2'), ('url3', 'cat1') ] response = self.client.create(["cat1", "cat2"], categoryPriors= [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.9}]) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels)) self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.post_gold_data(goldLabels)) self.assertEqual('OK', response['status']) response = self.client.post_compute() self.assertEqual('OK', response['status']) response = self.client.await_completion(self.client.get_workers_confusion_matrix()) pprint(response) response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost")) self.assertEqual('OK', response['status']) pprint(response) response = self.client.await_completion(self.client.get_workers_quality_payment()) pprint(response)
from client.gal import TroiaClient from nominalJobsTests.testSettings import * def check_status(res): if res['status'] != "OK" or res['result']['job_storage_status'] != "OK": print "FAILURE" if __name__ == "__main__": client = TroiaClient(ADDRESS) check_status(client.status()) client.create(CATEGORIES) client.await_completion(client.post_assigned_labels(ASSIGNED_LABELS)) client.await_completion(client.post_gold_data(GOLD_SAMPLES)) client.await_completion(client.post_compute()) # client.await_completion(client.get_predictions_objects()) # client.await_completion(client.get_prediction_workers_quality()) client.await_completion(client.get_prediction_zip()) client.delete() check_status(client.status())