class TestGoldLabels(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
response = self.client.create(CATEGORIES)
self.assertEqual("OK", response["status"])
def tearDown(self):
self.client.delete()
def _test_method(self, goldLabels):
response = self.client.await_completion(self.client.post_gold_data(goldLabels))
self.assertEqual("OK", response["status"])
# get the gold labels
response = self.client.await_completion(self.client.get_gold_data())
self.assertEqual("OK", response["status"])
result = response["result"]
self.assertEqual(len(goldLabels), len(result))
results = [tuple(receivedLabel.values()) for receivedLabel in response["result"]]
for label in goldLabels:
self.assertTrue(label in results)
def test_AddGetEmptyGoldLabels(self):
self._test_method([])
def test_AddGetGoldLabel_LongLabelName(self):
response = self.client.await_completion(
self.client.post_gold_data(
[
(
"sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs",
"notporn",
)
]
)
)
self.assertEqual("ERROR", response["status"])
self.assertEqual("Internal error: Object name should be shorter than 100 chars", response["result"])
def test_AddGetGoldLabel_PrintableASCII_RegularChars(self):
self._test_method([("url1", "notporn")])
def test_AddGetGoldLabel_PrintableASCII_SpecialChars(self):
self._test_method([("~!@#^&*()_+=-[]{}:<>,./", "notporn")])
def test_AddGetGoldLabel_ExtendedASCIIChars(self):
self._test_method([(u"™ž¤©", "notporn")])
def test_AddGetGoldLabel_UnicodeChars(self):
self._test_method([(u"ૉେஇ", "notporn")])
class NominalSimulation(Simulation):
def __init__(self, algorithm, *args, **kwargs):
super(NominalSimulation, self).__init__(*args, **kwargs)
self.algorithm = algorithm
def create(self):
self.tc = TroiaClient(TROIA_ADDRESS)
categories, init_kwargs = self.data_generator.gen_init_data()
self.tc.create(categories, algorithm=self.algorithm, **init_kwargs)
def _upload_golds(self, golds):
return ret_exectime(self.tc, self.tc.post_gold_data(golds))
def _upload_assigns_package(self, assigns):
return ret_exectime(self.tc, self.tc.post_assigned_labels(assigns))
def compute(self):
return ret_exectime(self.tc, self.tc.post_compute())
def shutdown(self):
self.tc.delete()
class TestWorkers(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def tearDown(self):
self.client.delete()
def test_AddGetEmptyWorkers(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS)
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_workers())
self.assertEqual('OK', response['status'])
self.assertEqual([], response['result'])
def test_AddGetWorkers_BeforeCompute(self):
self.client.create(CATEGORIES)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
response = self.client.await_completion(self.client.get_workers())
self.assertEqual(5, len(response['result']))
for w in response['result']:
self.assertEqual(5, w['value']['assigns'])
def test_AddGetWorkers_AfterCompute(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_compute())
#assigns check
response = self.client.await_completion(self.client.get_workers())
self.assertEqual(5, len(response['result']))
for w in response['result']:
self.assertEqual(5, w['value']['assigns'])
#confusion matrices check
response = self.client.await_completion(self.client.get_workers_confusion_matrix())
exp = {
u'worker1': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}],
u'worker3': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}],
u'worker2': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.},
{u'to': u'p**n', u'from': u'notporn', u'value': 0.3333}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.6666}],
u'worker5': [{u'to': u'p**n', u'from': u'p**n', u'value': 0.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 1.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}],
u'worker4': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}]}
for w in response['result']:
worker_name = w['workerName']
for e1 in w['value']:
exists = False
for e2 in exp[worker_name]:
if e2['to'] == e1['to'] and e2['from'] == e1['from']:
exists = True
self.assertAlmostEqual(
e2['value'],
e1['value'],
2,
"{}, from: {}, to: {}. expected:{}, was: {}".format(worker_name, e1['from'], e1['to'], e2['value'], e1['value']))
self.assertTrue(exists)
def test_GetWorkerInfo(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_worker_info('worker1'))
self.assertTrue(5, response['result']['value']['assigns'])
self.assertEqual(2, response['result']['value']['goldTests'])
self.assertEqual(1, response['result']['value']['correctGoldTests'])
self.assertTrue('worker1', response['result']['workerName'])
def test_GetWorkerAssigns(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_worker_assigns('worker1'))
print response
self.assertTrue(5, len(response['result']))
for a in response['result']:
self.assertTrue((a['worker'], a['object'], a['label']) in ASSIGNED_LABELS)
evaluationLabelsFile = os.path.join(testFolder, 'evaluation.txt')
for algorithm in algorithms:
print '--------'
print algorithm
client = TroiaClient(ADDRESS)
categories = dataProcessor.time_data_load_call(lambda: dataProcessor.loadCategories(categoriesFile))
goldLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadGoldLabels(goldLabelsFile))
evaluationLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadEvaluationLabels(evaluationLabelsFile))
assignedLabels = dataProcessor.time_data_load_call(lambda: dataProcessor.loadAssigns(assignsFile))
print 'Creating client'
createClient_t = dataProcessor.time_api_call(lambda: client.create(categories, algorithm=algorithm))
print 'Post gold labels'
loadGolds_t = dataProcessor.time_api_call(lambda: client.post_gold_data(goldLabels))
print 'Post evaluation labels'
loadEval_t = dataProcessor.time_api_call(lambda: client.post_evaluation_objects(evaluationLabels))
print 'Post assigned labels'
loadAssigns_t = dataProcessor.post_assigns(client, assignedLabels)
print 'Computing'
compute_t = dataProcessor.time_api_call(lambda: client.post_compute())
print 'Getting object prediction'
oPred_t = dataProcessor.time_api_call(lambda: client.get_objects_prediction("MinCost"))
print 'Getting worker quality summary'
wqEst_t = dataProcessor.time_api_call(lambda: client.get_workers_quality_estimated_summary())
class TestQualitySensitivePayments(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def tearDown(self):
self.client.delete()
def testQSP_ZeroWorkerQUality(self):
assignedLabels = [
('worker1', 'url1', 'cat2'),
('worker1', 'url2', 'cat1'),
('worker2', 'url1', 'cat1'),
('worker2', 'url2', 'cat1'),
('worker3', 'url1', 'cat1'),
('worker3', 'url2', 'cat2')]
goldLabels = [
('url1', 'cat1'),
('url2', 'cat2')
]
response = self.client.create(["cat1", "cat2"])
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_gold_data(goldLabels))
self.assertEqual('OK', response['status'])
response = self.client.post_compute()
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
self.assertEqual('OK', response['status'])
pprint(response)
response = self.client.await_completion(self.client.get_workers_quality_payment())
pprint(response)
def testQSP_NegativeWorkerQuality(self):
assignedLabels = [
('worker1', 'url1', 'cat2'),
('worker1', 'url2', 'cat1'),
('worker1', 'url3', 'cat2'),
('worker2', 'url1', 'cat1'),
('worker2', 'url2', 'cat1'),
('worker2', 'url3', 'cat1'),
('worker3', 'url1', 'cat1'),
('worker3', 'url2', 'cat2'),
('worker3', 'url3', 'cat1')
]
goldLabels = [
('url1', 'cat1'),
('url2', 'cat2'),
('url3', 'cat1')
]
response = self.client.create(["cat1", "cat2"], categoryPriors= [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.9}])
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_gold_data(goldLabels))
self.assertEqual('OK', response['status'])
response = self.client.post_compute()
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.get_workers_confusion_matrix())
pprint(response)
response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
self.assertEqual('OK', response['status'])
pprint(response)
response = self.client.await_completion(self.client.get_workers_quality_payment())
pprint(response)
from client.gal import TroiaClient
from nominalJobsTests.testSettings import *
def check_status(res):
if res['status'] != "OK" or res['result']['job_storage_status'] != "OK":
print "FAILURE"
if __name__ == "__main__":
client = TroiaClient(ADDRESS)
check_status(client.status())
client.create(CATEGORIES)
client.await_completion(client.post_assigned_labels(ASSIGNED_LABELS))
client.await_completion(client.post_gold_data(GOLD_SAMPLES))
client.await_completion(client.post_compute())
# client.await_completion(client.get_predictions_objects())
# client.await_completion(client.get_prediction_workers_quality())
client.await_completion(client.get_prediction_zip())
client.delete()
check_status(client.status())
