def createNominalJob():
client = TroiaClient(ADDRESS)
client.create(CATEGORIES)
jobs.append((client.jid, "NOMINAL"))
check_status(client, client.post_assigned_labels(ASSIGNED_LABELS))
check_status(client, client.post_evaluation_objects(EVALUATION_DATA))
check_status(client, client.post_compute())
check_assigns(client, len(ASSIGNED_LABELS))
class TestCommandStatus(unittest.TestCase):
def setUp(self):
self.tc = TroiaClient(ADDRESS)
def tearDown(self):
self.tc.delete()
def test_GetRedirectData(self):
self.tc.create(CATEGORIES)
assigend_labels_response = self.tc.get_assigned_labels()
time.sleep(0.5)
self.assertLessEqual(self.tc.get_status(assigend_labels_response['redirect'])['result'], [])
self.tc.await_completion(self.tc.post_assigned_labels(ASSIGNED_LABELS))
self.assertEqual(len(ASSIGNED_LABELS), len(self.tc.await_completion(self.tc.get_assigned_labels())['result']))
#still empty
self.assertListEqual(self.tc.get_status(assigend_labels_response['redirect'])['result'], [])
class NominalSimulation(Simulation):
def __init__(self, algorithm, *args, **kwargs):
super(NominalSimulation, self).__init__(*args, **kwargs)
self.algorithm = algorithm
def create(self):
self.tc = TroiaClient(TROIA_ADDRESS)
categories, init_kwargs = self.data_generator.gen_init_data()
self.tc.create(categories, algorithm=self.algorithm, **init_kwargs)
def _upload_golds(self, golds):
return ret_exectime(self.tc, self.tc.post_gold_data(golds))
def _upload_assigns_package(self, assigns):
return ret_exectime(self.tc, self.tc.post_assigned_labels(assigns))
def compute(self):
return ret_exectime(self.tc, self.tc.post_compute())
def shutdown(self):
self.tc.delete()
class TestPrediction(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
self.client.create(CATEGORIES, costMatrix=COST_MATRIX, categoryPriors=CATEGORY_PRIORS, algorithm=self.algorithm)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_evaluation_objects(EVALUATION_DATA))
self.client.await_completion(self.client.post_compute())
def tearDown(self):
self.client.delete()
def _getObjectsPrediction(self, method, expectedResults):
response = self.client.await_completion(self.client.get_objects_prediction(method))
self.assertEqual('OK', response['status'])
for categories in response['result']:
self.assertEqual(expectedResults[categories['objectName']], categories['categoryName'])
def _getEstimatedObjectsCost(self, costMethod, expectedCosts):
response = self.client.await_completion(self.client.get_estimated_objects_cost(costMethod))
self.assertEqual('OK', response['status'])
for cost in response['result']:
self.assertTrue(abs(expectedCosts[cost['objectName']] - cost['value']) / 100 < TOLERANCE)
def _getEvaluatedObjectsCost(self, labelChoosingMethod, expectedResults):
response = self.client.await_completion(self.client.get_evaluated_objects_cost(labelChoosingMethod))
self.assertEqual('OK', response['status'])
for item in response['result']:
self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE)
def _getEstimatedObjectsQuality(self, costAlgorithm, expectedDataQuality):
response = self.client.await_completion(self.client.get_estimated_objects_quality(costAlgorithm))
self.assertEqual('OK', response['status'])
for dataQuality in response['result']:
self.assertTrue(abs(expectedDataQuality[dataQuality['objectName']] - dataQuality['value']) / 100 < TOLERANCE)
def _getEvaluatedObjectsQuality(self, labelChoosingMethod, expectedResults):
response = self.client.await_completion(self.client.get_evaluated_objects_quality(labelChoosingMethod))
self.assertEqual('OK', response['status'])
for item in response['result']:
self.assertTrue(abs(expectedResults[item['objectName']] - item['value']) / 100 < TOLERANCE)
def _getEstimatedObjectsCostSummary (self, expectedObjectsCosts):
response = self.client.await_completion(self.client.get_objects_cost_estimated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedObjectsCosts.items():
self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEvaluatedObjectsCostSummary (self, expectedObjectsCosts):
response = self.client.await_completion(self.client.get_objects_cost_evaluated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedObjectsCosts.items():
self.assertTrue(abs(expectedObjectsCosts[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEstimatedObjectsQualitySummary(self, expectedDataQuality):
response = self.client.await_completion(self.client.get_objects_quality_estimated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedDataQuality.items():
self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEvaluatedObjectsQualitySummary(self, expectedDataQuality):
response = self.client.await_completion(self.client.get_objects_quality_evaluated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedDataQuality.items():
self.assertTrue(abs(expectedDataQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getEstimatedWorkerCost(self, costAlgorithm, expectedWorkerCost):
response = self.client.await_completion(self.client.get_estimated_workers_cost(costAlgorithm))
self.assertEqual('OK', response['status'])
for workerCost in response['result']:
self.assertTrue(abs(expectedWorkerCost[workerCost['workerName']] - workerCost['value']) / 100 < TOLERANCE)
def _getEstimatedWorkerQuality(self, costAlgorithm, expectedWorkerQuality):
response = self.client.await_completion(self.client.get_estimated_workers_quality(costAlgorithm))
self.assertEqual('OK', response['status'])
for workerQuality in response['result']:
self.assertTrue(abs(expectedWorkerQuality[workerQuality['workerName']] - workerQuality['value']) / 100 < TOLERANCE)
def _getEvaluatedWorkerQuality(self, costAlgorithm, expectedResults):
response = self.client.await_completion(self.client.get_evaluated_workers_quality(costAlgorithm))
self.assertEqual('OK', response['status'])
for item in response['result']:
self.assertTrue(abs(expectedResults[item['workerName']] - item['value']) / 100 < TOLERANCE)
def _getWorkersQualityEstimatedSummary(self, expectedWorkerQuality):
response = self.client.await_completion(self.client.get_workers_quality_estimated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedWorkerQuality.items():
self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getWorkersQualityEvaluatedSummary(self, expectedWorkerQuality):
response = self.client.await_completion(self.client.get_workers_quality_evaluated_summary())
self.assertEqual('OK', response['status'])
for k, v in expectedWorkerQuality.items():
self.assertTrue(abs(expectedWorkerQuality[k] - response['result'][k]) / 100 < TOLERANCE)
def _getCategoryProbability(self, expectedProbabilities):
for object in set((x[1] for x in ASSIGNED_LABELS)):
response = self.client.await_completion(self.client.get_probability_distribution(object))
self.assertEqual('OK', response['status'])
self.assertEqual(expectedProbabilities[object], response['result'])
class TestUnassignedLabels(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def get_priors(self, categories):
return [{'categoryName': c, "value": 1. / len(categories)} for c in categories]
def tearDown(self):
self.client.delete()
def _test_method(self, categories, priors, unassignedLabels, expectedProbabilities):
response = self.client.create(categories, categoryPriors=priors, algorithm="BMV")
self.assertEqual('OK', response['status'])
#post the unassigned labels
response = self.client.await_completion(self.client.post_objects(unassignedLabels))
self.assertEqual('OK', response['status'])
#get the unassigned labels
response = self.client.await_completion(self.client.get_objects())
self.assertEqual('OK', response['status'])
self.assertEqual(len(unassignedLabels), len(response['result']))
if (unassignedLabels):
result = response['result'][0]
self.assertEqual(unassignedLabels[0], result['name'])
def test_AddGetObjects_EmptyLabels(self):
self._test_method(CATEGORIES, CATEGORY_PRIORS, [], [])
def test_AddGetUnassignedLabels_LongLabelNames(self):
categories = ["category1", "category2"]
priors = [{"categoryName": "category1", "value": 0.0000000001}, {"categoryName": "category2", "value": 0.9999999999}]
unassignedLabels = ["sdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljssdgfdgfgfhdsjgfhgfhgfhhjhgjhjjghghkgkhjkfklsdjfkljs"]
expectedProbabilities = [('category1', 0.5), ('category2', 0.5)]
response = self.client.create(categories, categoryPriors=priors, algorithm="BMV")
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_objects(unassignedLabels))
self.assertEqual('ERROR', response['status'])
self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result'])
def test_AddGetUnassignedLabels_PrintableASCII_RegularChars(self):
categories = ["category1", "category2", "category3", "category4"]
unassignedLabels = ["testObject1"]
expectedProbabilities = [('category1', 0.25),
('category2', 0.25),
('category3', 0.25),
('category4', 0.25)]
self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)
def test_AddGetUnassignedLabels_PrintableASCII_SpecialChars(self):
categories = ["category1", "category2", "category3"]
unassignedLabels = ["~!@#%^&*()_+=-[]{}:<>,./"]
expectedProbabilities = [('category1', 0.3333333333333333),
('category2', 0.3333333333333333),
('category3', 0.3333333333333333)]
self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)
def test_AddGetUnassignedLabels_ExtendedASCIIChars(self):
categories = ["category1", "category2"]
unassignedLabels = [u"ëñµ¼Úæ"]
expectedProbabilities = [('category1', 0.5), ('category2', 0.5)]
self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)
def test_AddGetUnassignedLabels_UnicodeChars(self):
categories = ["category1", "category2"]
unassignedLabels = [u"ూഹܬआਖ਼"]
expectedProbabilities = [('category1', 0.5), ('category2', 0.5)]
self._test_method(categories, self.get_priors(categories), unassignedLabels, expectedProbabilities)
def test_GetObjectAssigns(self):
response = self.client.create(CATEGORIES)
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.assertEqual('OK', response['status'])
#post the unassigned label
unassignedLabel = [u"ూഹܬआਖ਼"]
response = self.client.await_completion(self.client.post_objects(unassignedLabel))
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.get_objects())
self.assertEqual('OK', response['status'])
result = response['result']
self.assertEqual(6, len(result))
results = []
for object in result:
if object['name'] not in unassignedLabel:
assigns = self.client.await_completion(self.client.get_object_assigns(object['name']))['result']
for a in assigns:
results.append((a['worker'], a['object'], a['label']))
for label in ASSIGNED_LABELS:
self.assertTrue(label in results)
def test_GetObjectInfo(self):
response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS)
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels([('worker1', 'url1', 'p**n'), ('worker1', 'url2', 'p**n')]))
self.assertEqual('OK', response['status'])
#post the unassigned label
objects_without_assigns = ["newUnassignedLabel"]
response = self.client.await_completion(self.client.post_objects(objects_without_assigns))
self.assertEqual('OK', response['status'])
#get labels
response = self.client.await_completion(self.client.get_objects())
self.assertEqual('OK', response['status'])
result = response['result']
self.assertEqual(3, len(result))
self.client.await_completion(self.client.post_compute())
for label in result:
response = self.client.await_completion(self.client.get_probability_distribution(label['name']))
dist = response['result'][0]
self.assertAlmostEqual(dist['value'], 0.5 if label['name'] in objects_without_assigns else 0.9 if dist['categoryName'] == 'p**n' else 0.1)
class TestWorkers(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def tearDown(self):
self.client.delete()
def test_AddGetEmptyWorkers(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS)
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_workers())
self.assertEqual('OK', response['status'])
self.assertEqual([], response['result'])
def test_AddGetWorkers_BeforeCompute(self):
self.client.create(CATEGORIES)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
response = self.client.await_completion(self.client.get_workers())
self.assertEqual(5, len(response['result']))
for w in response['result']:
self.assertEqual(5, w['value']['assigns'])
def test_AddGetWorkers_AfterCompute(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_compute())
#assigns check
response = self.client.await_completion(self.client.get_workers())
self.assertEqual(5, len(response['result']))
for w in response['result']:
self.assertEqual(5, w['value']['assigns'])
#confusion matrices check
response = self.client.await_completion(self.client.get_workers_confusion_matrix())
exp = {
u'worker1': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}],
u'worker3': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}],
u'worker2': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.},
{u'to': u'p**n', u'from': u'notporn', u'value': 0.3333}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.6666}],
u'worker5': [{u'to': u'p**n', u'from': u'p**n', u'value': 0.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 1.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 1.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 0.0}],
u'worker4': [{u'to': u'p**n', u'from': u'p**n', u'value': 1.0}, {u'to': u'notporn', u'from': u'p**n', u'value': 0.0},
{u'to': u'p**n', u'from': u'notporn', u'value': 0.0}, {u'to': u'notporn', u'from': u'notporn', u'value': 1.0}]}
for w in response['result']:
worker_name = w['workerName']
for e1 in w['value']:
exists = False
for e2 in exp[worker_name]:
if e2['to'] == e1['to'] and e2['from'] == e1['from']:
exists = True
self.assertAlmostEqual(
e2['value'],
e1['value'],
2,
"{}, from: {}, to: {}. expected:{}, was: {}".format(worker_name, e1['from'], e1['to'], e2['value'], e1['value']))
self.assertTrue(exists)
def test_GetWorkerInfo(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_gold_data(GOLD_SAMPLES))
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_worker_info('worker1'))
self.assertTrue(5, response['result']['value']['assigns'])
self.assertEqual(2, response['result']['value']['goldTests'])
self.assertEqual(1, response['result']['value']['correctGoldTests'])
self.assertTrue('worker1', response['result']['workerName'])
def test_GetWorkerAssigns(self):
self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.client.await_completion(self.client.post_assigned_labels(ASSIGNED_LABELS))
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_worker_assigns('worker1'))
print response
self.assertTrue(5, len(response['result']))
for a in response['result']:
self.assertTrue((a['worker'], a['object'], a['label']) in ASSIGNED_LABELS)
class TestCachedScheduler(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
self.scheduler = 'CachedScheduler'
self.utils = TestUtils()
def tearDown(self):
self.client.delete()
def _createTestPrereq(self, algorithm, scheduler, calculator, assigns, categories = CATEGORIES, categoryPriors = CATEGORY_PRIORS, costMatrix = COST_MATRIX):
response = self.client.create(
categories,
categoryPriors=categoryPriors,
algorithm=algorithm,
scheduler=scheduler,
prioritycalculator=calculator,
costMatrix=costMatrix
)
self.assertEqual('OK', response['status'])
self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(assigns))['status'])
self.assertEqual('OK', self.client.await_completion(self.client.post_compute())['status'])
def getObjectCountsList(self, assigns, reverse=False, excludedObjectsList=None):
#create the dictionary containing the objects and the associated no of assigns
labelsDict = {}
for l in assigns:
if l[1] in labelsDict.keys():
labelsDict[l[1]] += 1
else:
labelsDict[l[1]] = 1
#sort the objects ascending, based on the no of labels
sortedList = sorted(labelsDict.items(), key=itemgetter(1), reverse=reverse)
if (excludedObjectsList):
for value in sortedList:
if value[0] in excludedObjectsList:
sortedList.remove(value)
return sortedList
def getObjectCostsList(self):
response = self.client.await_completion(self.client.get_estimated_objects_cost("ExpectedCost"))
self.assertEquals("OK", response['status'])
objectCosts = {}
for result in response['result']:
objectCosts[result['objectName']] = result['value']
#sort the objects descending, based on cost
objectCostList = sorted(objectCosts.items(), key=itemgetter(1), reverse=True)
return objectCostList
def getAssignedLabels(self):
response = self.client.await_completion(self.client.get_assigned_labels())
self.assertEqual('OK', response['status'])
assignedLabels = [(l['worker'], l['object'], l['label']) for l in response['result']]
return assignedLabels
def _runScheduler(self, workerId=None):
if (workerId):
return self.client.get_next_worker_object(workerId)
else:
return self.client.get_next_object()
def _runTestMethod(self, calculator, expectedObjectList, newAssign, workerId=None, excludedObjectsList=None):
for (_, objectCost) in expectedObjectList:
response = self.client.await_completion(self._runScheduler(workerId))
objectName = response['result']['name']
#get the objects with equal costs
equalCostObjects = [item[0] for item in expectedObjectList
if item[1] == objectCost]
self.assertTrue(objectName in equalCostObjects)
# This one should be null. That means the 'result' key is not present in the response.
response = self.client.await_completion(self._runScheduler(workerId))
self.assertIsNone(response.get('result', None))
# Add assign to the object. The object should be returned by subsequent 'nextObject' call.
self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(newAssign))['status'])
if calculator == 'countassigns':
newObjectsList = self.getObjectCountsList(newAssign, False, excludedObjectsList)
else:
self.assertEqual('OK', self.client.await_completion(self.client.post_compute())['status'])
newObjectsList = self.getObjectCostsList()
#get the objects having the maximum priority
maxPriorityObjects = [item[0] for item in newObjectsList
if item[1] == newObjectsList[0][1]]
response = self.client.await_completion(self._runScheduler(workerId))
self.assertEqual('OK',response['status'])
if response.get('result') == None:
self.assertTrue(len(newAssign) == 0)
else:
self.assertTrue(response['result']['name'] in maxPriorityObjects)
@unittest.skip('Skipping test')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddNewAssigns(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'notporn'),
('worker1', 'object2', 'p**n'),
('worker2', 'object2', 'notporn'),
('worker3', 'object2', 'p**n')]
expectedObjectsList = self.getObjectCountsList(assigns, False)
newAssigns = [('worker1', 'object3', 'p**n')]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, newAssigns)
@unittest.skip('Skipping test')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddEmptyAssign(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker1', 'object2', 'p**n'),
('worker2', 'object2', 'p**n')]
expectedObjectsList = self.getObjectCountsList(assigns, False)
newAssign = []
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, newAssign)
@unittest.skip('Skipping test')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextObject_SameLabelCounts_AddNewAssign(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'p**n'),
('worker1', 'object2', 'p**n'),
('worker2', 'object2', 'p**n')]
expectedObjectsList = self.getObjectCountsList(assigns, False)
newAssign = [('worker3', 'object0', 'notporn')]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, newAssign)
@unittest.skip('Skipping test')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextObject_SameLabelCounts_AddEmptyAssign(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'p**n'),
('worker1', 'object2', 'p**n'),
('worker2', 'object2', 'p**n')]
expectedObjectsList = self.getObjectCountsList(assigns, False)
newAssign = []
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, newAssign)
@unittest.skip('Skipping test')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CostBasedCalculator_GetNextObject_SameObjectCosts(self, algorithm):
calculator = 'costbased'
categories = ["cat1", "cat2"]
categoryPriors = [{"categoryName": "cat1", "value": 0.5}, {"categoryName": "cat2", "value": 0.5}]
costMatrix = [{"from": "cat1", "to": "cat2", "value": 1.0}, {"from": "cat1", "to": "cat1", "value": 0.0},
{"from": "cat2", "to": "cat1", "value": 1.0}, {"from": "cat2", "to": "cat2", "value": 0.0}]
assigns = [('worker1', 'object1', 'cat1'),
('worker2', 'object1', 'cat1'),
('worker3', 'object2', 'cat1'),
('worker4', 'object2', 'cat1')]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors, costMatrix)
objectCostList = self.getObjectCostsList()
newAssign = [('worker3', 'object0', 'cat1')]
self._runTestMethod(calculator, objectCostList, newAssign)
@unittest.skip('Skipping test')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CostBasedCalculator_GetNextObject_DifferentObjectCosts(self, algorithm):
calculator = 'costbased'
categories = ["p**n", "notporn"]
categoryPriors = [{"categoryName": "p**n", "value": 0.1}, {"categoryName": "notporn", "value": 0.9}]
costMatrix = [{"from": "p**n", "to": "notporn", "value": 1.0}, {"from": "p**n", "to": "p**n", "value": 0.0},
{"from": "notporn", "to": "p**n", "value": 1.0}, {"from": "notporn", "to": "notporn", "value": 0.0}]
assigns = [('worker0', 'object0', 'notporn'),
('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'p**n')]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors, costMatrix)
objectCostList = self.getObjectCostsList()
newAssign = [('worker3', 'object2', 'p**n')]
self._runTestMethod(calculator, objectCostList, newAssign)
@unittest.skip('Skipping test')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CostBasedCalculator_GetNextObject_DifferentObjectCosts_AddEmptyLabel(self, algorithm):
calculator = 'costbased'
categories = ["p**n", "notporn"]
categoryPriors = [{"categoryName": "p**n", "value": 0.1}, {"categoryName": "notporn", "value": 0.9}]
costMatrix = [{"from": "p**n", "to": "notporn", "value": 1.0}, {"from": "p**n", "to": "p**n", "value": 0.0},
{"from": "notporn", "to": "p**n", "value": 1.0}, {"from": "notporn", "to": "notporn", "value": 0.0}]
assigns = [('worker0', 'object0', 'notporn'),
('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'p**n')]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors, costMatrix)
objectCostList = self.getObjectCostsList()
newAssign = []
self._runTestMethod(calculator, objectCostList, newAssign)
@unittest.skip('Skipping test')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextWorkerObject_SameLabelCounts(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker2', 'object2', 'notporn'),
('worker2', 'object3', 'notporn'),
('worker3', 'object2', 'notporn'),
('worker3', 'object3', 'notporn')
]
expectedObjectsList = [('object2', 2), ('object3', 2)]
newAssign = [('worker4', 'object2', 'notporn')]
excludedObjectsList = ['object1']
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, newAssign, 'worker1', excludedObjectsList)
@unittest.skip('Skipping test')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextWorkerObject_DifferentLabelCounts(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'p**n'),
('worker2', 'object2', 'notporn'),
('worker3', 'object2', 'notporn'),
('worker3', 'object3', 'p**n')]
expectedObjectsList = [('object3', 1), ('object2', 2)]
newAssign = [('worker4', 'object2', CATEGORIES[1])]
excludedObjectsList = ['object1']
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, newAssign, 'worker1', excludedObjectsList)
class TestNormalScheduler(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
self.scheduler = 'normalscheduler'
self.utils = TestUtils()
def tearDown(self):
self.client.delete()
def _createTestPrereq(self, algorithm, scheduler, calculator, assigns, categories = CATEGORIES, categoryPriors = CATEGORY_PRIORS):
response = self.client.create(
categories,
categoryPriors=categoryPriors,
algorithm=algorithm,
scheduler=scheduler,
prioritycalculator=calculator
)
self.assertEqual('OK', response['status'])
self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(assigns))['status'])
self.assertEqual('OK', self.client.await_completion(self.client.post_compute())['status'])
def getObjectCountsList(self, assigns, reverse=False, excludedObjectsList=False):
#create the dictionary containing the objects and the associated no of assigns
labelsDict = {}
for l in assigns:
if l[1] in labelsDict.keys():
labelsDict[l[1]] += 1
else:
labelsDict[l[1]] = 1
#sort the objects ascending, based on the no of labels
sortedList = sorted(labelsDict.items(), key=itemgetter(1), reverse=reverse)
if (excludedObjectsList):
for value in sortedList:
if value[0] in excludedObjectsList:
sortedList.remove(value)
return sortedList
def getObjectCostsList(self):
response = self.client.await_completion(self.client.post_compute())
self.assertEquals("OK", response['status'])
response = self.client.await_completion(self.client.get_estimated_objects_cost("ExpectedCost"))
self.assertEquals("OK", response['status'])
objectCosts = {}
for result in response['result']:
objectCosts[result['objectName']] = result['value']
#sort the objects descending, based on cost
objectCostList = sorted(objectCosts.items(), key=itemgetter(1), reverse=True)
return objectCostList
def getAssignedLabels(self):
response = self.client.await_completion(self.client.get_assigned_labels())
self.assertEqual('OK', response['status'])
assignedLabels = [(l['worker'], l['object'], l['label']) for l in response['result']]
return assignedLabels
def _runScheduler(self, workerId=None):
if (workerId):
return self.client.get_next_worker_object(workerId)
else:
return self.client.get_next_object()
def _runTestMethod(self, calculator, expectedObjectList, newAssign, workerId=None, excludedObjectsList=None):
for i in xrange(len(expectedObjectList)):
response = self.client.await_completion(self._runScheduler(workerId))
self.assertTrue(response['result']['name'] in expectedObjectList)
# Add assign to the object. The object should be returned by subsequent 'nextObject' call.
self.assertEqual('OK', self.client.await_completion(self.client.post_assigned_labels(newAssign))['status'])
response = self.client.await_completion(self._runScheduler(workerId))
self.assertEqual('OK',response['status'])
if calculator == 'countassigns':
newObjectsList = self.getObjectCountsList(self.getAssignedLabels(), False, excludedObjectsList)
else:
newObjectsList = self.getObjectCostsList()
#get the objects having the maximum priority
maxPriorityObjects = [item[0] for item in newObjectsList
if item[1] == newObjectsList[0][1]]
response = self.client.await_completion(self._runScheduler(workerId))
self.assertEqual('OK',response['status'])
self.assertTrue(response['result']['name'] in maxPriorityObjects)
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddEmptyAssigns(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'notporn'),
('worker2', 'object2', 'notporn')]
sortedList = self.getObjectCountsList(assigns, False)
minValue = sortedList[0][1]
expectedObjectsList = [o[0] for o in sortedList if o[1] == minValue]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, [])
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextObject_DifferentLabelCounts_AddNewAssigns(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'p**n'),
('worker1', 'object2', 'p**n'),
('worker2', 'object2', 'notporn'),
('worker3', 'object2', 'p**n'),
]
sortedList = self.getObjectCountsList(assigns, False)
minValue = sortedList[0][1]
expectedObjectsList = [o[0] for o in sortedList if o[1] == minValue]
newAssign = [('worker1', 'object3', 'p**n')]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, newAssign)
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextObject_SameLabelCounts(self, algorithm):
calculator = 'countassigns'
categories = ["cat1", "cat2"]
categoryPriors = [{"categoryName": "cat1", "value": 0.5}, {"categoryName": "cat2", "value": 0.5}]
assigns = [('worker1', 'object1', 'cat1'),
('worker1', 'object2', 'cat1'),
('worker2', 'object1', 'cat2'),
('worker2', 'object2', 'cat2')]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors)
sortedList = self.getObjectCountsList(assigns, False)
minValue = sortedList[0][1]
expectedObjectList = [o[0] for o in sortedList if o[1] == minValue]
newAssign = [('worker3', 'object3', 'cat1')]
self._runTestMethod(calculator, expectedObjectList, newAssign)
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CostBasedCalculator_GetNextObject_SameCosts(self, algorithm):
calculator = 'costbased'
categories = ["cat1", "cat2"]
categoryPriors = [{"categoryName": "cat1", "value": 0.5}, {"categoryName": "cat2", "value": 0.5}]
assigns = [('worker1', 'object1', 'cat1'),
('worker2', 'object1', 'cat1'),
('worker3', 'object2', 'cat1'),
('worker4', 'object2', 'cat1')]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors)
sortedList = self.getObjectCostsList()
minValue = sortedList[0][1]
expectedObjectList = [o[0] for o in sortedList if o[1] == minValue]
newAssign = [('worker5', 'object3', 'cat1')]
self._runTestMethod(calculator, expectedObjectList, newAssign)
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CostBasedCalculator_GetNextObject_DifferentCosts(self, algorithm):
calculator = 'costbased'
categories = ["cat1", "cat2", "cat3"]
categoryPriors = [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.3}, {"categoryName": "cat3", "value": 0.6}]
assigns = [('worker1', 'object1', 'cat1'),
('worker2', 'object1', 'cat2'),
('worker2', 'object2', 'cat1'),
('worker3', 'object1', 'cat1'),
('worker3', 'object2', 'cat2')]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns, categories, categoryPriors)
sortedList = self.getObjectCostsList()
minValue = sortedList[0][1]
expectedObjectList = [o[0] for o in sortedList if o[1] == minValue]
newAssign = [('worker4', 'object0', 'cat3')]
self._runTestMethod(calculator, expectedObjectList, newAssign)
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextWorkerObject_DifferentLabelCounts(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'p**n'),
('worker2', 'object2', 'notporn'),
('worker3', 'object2', 'notporn'),
('worker4', 'object2', 'notporn'),
('worker3', 'object3', 'p**n')
]
expectedObjectsList = ['object3']
newAssigns = [('worker4', 'object3', CATEGORIES[0])]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, newAssigns, 'worker1')
@data('BDS', 'IDS', 'BMV', 'IMV')
def test_CountAssignsCalculator_GetNextWorkerObject_SameLabelCounts(self, algorithm):
calculator = 'countassigns'
assigns = [('worker1', 'object1', 'p**n'),
('worker2', 'object1', 'p**n'),
('worker2', 'object2', 'notporn'),
('worker3', 'object2', 'p**n'),
('worker3', 'object3', 'notporn'),
('worker4', 'object3', 'p**n')
]
expectedObjectsList = ['object3', 'object2']
newAssigns = [('worker4', 'object4', CATEGORIES[0])]
self._createTestPrereq(algorithm, self.scheduler, calculator, assigns)
self._runTestMethod(calculator, expectedObjectsList, newAssigns, 'worker1')
class TestAssignedLabels(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def tearDown(self):
self.client.delete()
def _test_method(self, assigned_labels):
#post the assigned labels
response = self.client.await_completion(self.client.post_assigned_labels(assigned_labels))
self.assertEqual('OK', response['status'])
#get the assigned labels
response = self.client.await_completion(self.client.get_assigned_labels())
self.assertEqual('OK', response['status'])
self.assertEqual(len(response['result']), len(assigned_labels))
results = [(l['worker'], l['object'], l['label']) for l in response['result']]
for label in assigned_labels:
self.assertTrue(label in results)
def test_AddGetEmptyAssignedLabels(self):
response = self.client.create(CATEGORIES, categoryPriors=CATEGORY_PRIORS, costMatrix=COST_MATRIX)
self.assertEqual('OK', response['status'])
self._test_method([])
def test_AddAssignedLabelsWithInvalidCategory(self):
categories = ["category1", "category2"]
response = self.client.create(categories)
self.assertEqual('OK', response['status'])
#post the assigned labels
assignedLabels = [('worker1', 'url1', 'category3')]
response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
self.assertEqual('ERROR', response['status'])
#get the assigned labels
response = self.client.await_completion(self.client.get_assigned_labels())
self.assertEqual('OK', response['status'])
result = response['result']
self.assertFalse(result)
def test_AddGetAssignedLabels_LongWorkerNames(self):
categories = ['category1', "category2"]
response = self.client.create(categories)
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels([('hjkdhfhdfgjkshfghdsgffgfhfghgjhghjgjgjgjgjgjgjldkgjhjkdhfhdfgjkshfghdsgffgfhfghgjhghjgjgjgjgjgjgjldkg', 'object_dsjgfhgfhgfhhjdfhgkgkhjkfklsdjfkljsdfj', 'category1')]))
self.assertEqual('ERROR', response['status'])
self.assertEqual('Internal error: Worker name should be shorter than 100 chars', response['result'])
def test_AddGetAssignedLabels_LongObjectNames(self):
categories = ['category1', "category2"]
response = self.client.create(categories)
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels([('hjkdhfhdfgjkshfghdsgffgfhfghgjhghjgkgj', 'object_dsjgfhgfhgfhhjdfhgjghghkgkhjkfklsdjfkljsdfjobjectdsjgfhgfhgfhhjdfhgjghghkgkhjkfklsdjfkljsdfjwd', 'category1')]))
self.assertEqual('ERROR', response['status'])
self.assertEqual('Internal error: Object name should be shorter than 100 chars', response['result'])
def test_AddGetAssignedLabels_PrintableASCII_RegularChars(self):
response = self.client.create(CATEGORIES)
self.assertEqual('OK', response['status'])
self._test_method(ASSIGNED_LABELS)
def test_AddGetAssignedLabels_PrintableASCII_SpecialChars(self):
categories = ['category1', "category2", "category3"]
response = self.client.create(categories)
self.assertEqual('OK', response['status'])
self._test_method([('a%a', '~!@#%^&*()_+=-[]{}:<>,./', 'category1'),
('b%%b', '%%%', 'category2'),
('c%%!<>c', '~!@#^&*[](){}-_+=<>?/.,:', 'category3')])
def test_AddGetAssignedLabels_ExtendedASCIIChars(self):
categories = ['category1', "category2", "category3"]
response = self.client.create(categories)
self.assertEqual('OK', response['status'])
self._test_method([(u'®¶', u'œŒ', 'category1'),
(u'ÀÆË', u'™ž¤©', 'category2'),
(u'ëñ', u'µ¼Úæ', 'category3')])
def test_AddGetAssignedLabels_UnicodeChars(self):
categories = ['category1', "category2", "category3"]
response = self.client.create(categories)
self.assertEqual('OK', response['status'])
self._test_method([(u'ૉେஇ', u'ΨҖӖմڅ', 'category1'),
(u'ూഹ', u'ܬआਖ਼', 'category2')])
class TestQualitySensitivePayments(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def tearDown(self):
self.client.delete()
def testQSP_ZeroWorkerQUality(self):
assignedLabels = [
('worker1', 'url1', 'cat2'),
('worker1', 'url2', 'cat1'),
('worker2', 'url1', 'cat1'),
('worker2', 'url2', 'cat1'),
('worker3', 'url1', 'cat1'),
('worker3', 'url2', 'cat2')]
goldLabels = [
('url1', 'cat1'),
('url2', 'cat2')
]
response = self.client.create(["cat1", "cat2"])
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_gold_data(goldLabels))
self.assertEqual('OK', response['status'])
response = self.client.post_compute()
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
self.assertEqual('OK', response['status'])
pprint(response)
response = self.client.await_completion(self.client.get_workers_quality_payment())
pprint(response)
def testQSP_NegativeWorkerQuality(self):
assignedLabels = [
('worker1', 'url1', 'cat2'),
('worker1', 'url2', 'cat1'),
('worker1', 'url3', 'cat2'),
('worker2', 'url1', 'cat1'),
('worker2', 'url2', 'cat1'),
('worker2', 'url3', 'cat1'),
('worker3', 'url1', 'cat1'),
('worker3', 'url2', 'cat2'),
('worker3', 'url3', 'cat1')
]
goldLabels = [
('url1', 'cat1'),
('url2', 'cat2'),
('url3', 'cat1')
]
response = self.client.create(["cat1", "cat2"], categoryPriors= [{"categoryName": "cat1", "value": 0.1}, {"categoryName": "cat2", "value": 0.9}])
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.post_gold_data(goldLabels))
self.assertEqual('OK', response['status'])
response = self.client.post_compute()
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.get_workers_confusion_matrix())
pprint(response)
response = self.client.await_completion(self.client.get_estimated_workers_quality("ExpectedCost"))
self.assertEqual('OK', response['status'])
pprint(response)
response = self.client.await_completion(self.client.get_workers_quality_payment())
pprint(response)
from client.gal import TroiaClient
from nominalJobsTests.testSettings import *
def check_status(res):
if res['status'] != "OK" or res['result']['job_storage_status'] != "OK":
print "FAILURE"
if __name__ == "__main__":
client = TroiaClient(ADDRESS)
check_status(client.status())
client.create(CATEGORIES)
client.await_completion(client.post_assigned_labels(ASSIGNED_LABELS))
client.await_completion(client.post_gold_data(GOLD_SAMPLES))
client.await_completion(client.post_compute())
# client.await_completion(client.get_predictions_objects())
# client.await_completion(client.get_prediction_workers_quality())
client.await_completion(client.get_prediction_zip())
client.delete()
check_status(client.status())
class BugRegressionTests(unittest.TestCase):
def setUp(self):
self.client = TroiaClient(ADDRESS)
def tearDown(self):
self.client.delete()
def test_GH_83(self):
"""
Test for github issue https://github.com/ipeirotis/Troia-Server/issues/83
Thanks to PeccatorImpius for reporting
"""
categories = ['yes', 'no', 'blank']
response = self.client.create(categories,
prioritycalculator="CostBased",
costMatrix=[{"value":5.0, "to":"blank", "from":"blank"},
{"value":5.0, "to":"no", "from":"blank"},
{"value":5.0, "to":"yes", "from":"blank"},
{"value":1.0, "to":"blank", "from":"no"},
{"value":0.0, "to":"no", "from":"no"},
{"value":1.0, "to":"yes", "from":"no"},
{"value":1.0, "to":"blank", "from":"yes"},
{"value":1.0, "to":"no", "from":"yes"},
{"value":0.0, "to":"yes", "from":"yes"}],
epsilon=1.0E-4,
iterations=10,
algorithm='IDS',
scheduler='NormalScheduler')
self.assertEqual('OK', response['status'])
assignedLabels = [
('A2G061ZJVQXGE', '21K74J79KQW6KZ5ZI8OFKD455U4HJ6', 'blank'),
('A2G061ZJVQXGE', '2FWPLSP75KLMPN1Y0O8QMQVJZMCZTJ', 'yes'),
('A224TK7J4KA1LV', '283PHIT3HVWAEB01E1AQA1727RZCKR', 'yes'),
('A224TK7J4KA1LV', '2S3YHVI44OS23CC3P15Z0BAPDTU4Y4', 'yes'),
('A224TK7J4KA1LV', '2NCXFCD45XCECEVNZ7BIV3TGXMSYWJ', 'yes'),
('A224TK7J4KA1LV', '295QQ7Q670519BUI3FTTYC1XUY7WC6', 'no'),
('A224TK7J4KA1LV', '255NVRH1KHO9X796JKEVHLKWFFDZQ5', 'yes'),
('A224TK7J4KA1LV', '21Z1Q6SVQ8H8RD8PIZWOJVCI99NN4U', 'yes'),
('A224TK7J4KA1LV', '2JGU98JHSTLBMCIANX0IWBJER94EQE', 'yes'),
('A224TK7J4KA1LV', '2Z3KH1Q6SVQ80ZQH28XLDOBVK0S2LC', 'yes'),
('A224TK7J4KA1LV', '2P3Z6R70G5R9R4QEF9574SSTVG2P9O', 'yes'),
('A224TK7J4KA1LV', '25UCF0CP5KXPC95DUBPGCPM7VGWWX9', 'yes'),
('A224TK7J4KA1LV', '2AOYTWX4H3H282M8LN7IEIJRJNW4ZT', 'no'),
('A224TK7J4KA1LV', '2J5JQ2172Z9998O6WZ3VMBU1B1FZR6', 'no'),
('A224TK7J4KA1LV', '2SIHFL42J1LYOZT2ELD0QGFKPVS2NE', 'yes'),
('A224TK7J4KA1LV', '2W36VCPWZ9RNWN0J7FJ70N3D1YKPA3', 'yes'),
('A224TK7J4KA1LV', '2F8S1CSTMOFXA4AZUQSE6DNXRXCROW', 'yes'),
('A224TK7J4KA1LV', '2TRKSSZVXX2QNWCPW4FYONIW95IN5Z', 'no'),
('A224TK7J4KA1LV', '2L13NAB6BFMFYFGTU7STRFDGMA2HU2', 'yes'),
('A224TK7J4KA1LV', '2B3RGNTBJ3MMWONA1LE6AKHBH439VT', 'yes'),
('A224TK7J4KA1LV', '24ZMVHVKCJ01KBK2GXY9SQW69QD758', 'yes'),
('A224TK7J4KA1LV', '2V99Y8TNKIMZBDNFRVJ2WM0POD59W3', 'yes'),
('A224TK7J4KA1LV', '2FK4E6AUBXHWB8PAXN0VJZWDZ3NX1B', 'yes'),
('A224TK7J4KA1LV', '2WAVGULF395SFX5BDTJ3VYNJJG5PBY', 'yes'),
('A224TK7J4KA1LV', '2DMZMAZHHRRLYHIJZ1MRP1KHWR4CLU', 'yes'),
('A224TK7J4KA1LV', '27E618N46UXFV4M09Q5TVDSN128RPD', 'yes'),
('A224TK7J4KA1LV', '2J9CJK63ZVE30IIF6W1QHJL5ZL59Y2', 'yes'),
('A224TK7J4KA1LV', '2DS6X3YOCCF0VGNF93KIVIIX6YURQ4', 'yes'),
('A224TK7J4KA1LV', '23ZGYQ4J3NABP2XHRTFY6IT1115PC5', 'yes'),
('A224TK7J4KA1LV', '2PFMB3Q39Z0BPLBJI7VE3STVZ26UJK', 'yes'),
('A224TK7J4KA1LV', '283FQ0ONNVRHKBZJLS7RJ76N3RBULM', 'no'),
('A224TK7J4KA1LV', '2F8S1CSTMOFXA4AZUQSE6DNXRXFORW', 'yes'),
('A224TK7J4KA1LV', '21UJ011K274JQ02L8KS8V46U5X5BD9', 'no'),
('A224TK7J4KA1LV', '2QAKMT6YTWX40UZX1PVDH9GIE0C0VB', 'yes'),
('A224TK7J4KA1LV', '2C18EBDPGFL6E37RBNINLWIO3JLH2M', 'yes'),
('A224TK7J4KA1LV', '2Y6VR14IXKO240KCQQNAEX3YWU59AV', 'yes'),
('A224TK7J4KA1LV', '2P7JWKUDD8XMUU8YLDRBEUTOEBXMB2', 'yes'),
('A224TK7J4KA1LV', '2525UFJ51Y7QXFHAA4J1KSTMWXQ9CX', 'yes'),
('A224TK7J4KA1LV', '2NCXFCD45XCECEVNZ7BIV3TGXMQWYF', 'yes'),
('A224TK7J4KA1LV', '27ZM8JIA0RYN20KIVYD0DBMJ1C8507', 'yes'),
('A224TK7J4KA1LV', '2I63W5XH0JPPB9KFASGL0P75S3FEKP', 'yes'),
('A224TK7J4KA1LV', '21RRJ85OZIZE0DS510FOY5EWXFB9XW', 'yes'),
('A224TK7J4KA1LV', '2LAMEZZ2MIKM60C31NP3GTWW50TN6F', 'yes'),
('A224TK7J4KA1LV', '241KM05BMJTU0PEXS7G9ZA7SDEFEJQ', 'yes'),
('A224TK7J4KA1LV', '2UZDM25L8I9NINOCQDXOM4QWFBYHRY', 'yes'),
('A224TK7J4KA1LV', '2J1E1M7PKK9LANOJBO30CLZXX3YZSU', 'yes'),
('A224TK7J4KA1LV', '2LWGDHDM25L8105U8K8E76OEC8MEO6', 'no'),
('A224TK7J4KA1LV', '2XANTL0XULRG6KTEF0DD55FPF5X2OT', 'yes'),
('A224TK7J4KA1LV', '2FC98JHSTLB34RX6VN9OJJEJZWORFD', 'yes'),
('A224TK7J4KA1LV', '2D2UDD8XMB3QM0HVNKLTW6T4MDIEPN', 'yes'),
('A224TK7J4KA1LV', '2C73TVOK5UFJOSG22SFZNYQS9UI743', 'yes'),
('A224TK7J4KA1LV', '25XXRDS4IC1EH45SVTD19A2I3GUWZ6', 'no'),
('A224TK7J4KA1LV', '2PG1THV8ER9YRK5FU0QSU5KFPAT2PA', 'yes'),
('A224TK7J4KA1LV', '2C79746OQ1SN9HPLILR59QKCV9RN39', 'yes'),
('A224TK7J4KA1LV', '2YDKCJ011K27NAP4W4N698N4ECQ9BQ', 'yes'),
('A224TK7J4KA1LV', '2XGQ4J3NAB6BYDXA0CPI11TJNV6ER,F', 'yes'),
('A224TK7J4KA1LV', '23K5L8I9NZW605H10SVQ47T8ER6UKJ', 'yes'),
('A224TK7J4KA1LV', '2ZCUU00OQKKAN90NK6TUA680MEQWWG', 'yes'),
('A224TK7J4KA1LV', '2P1HSTLB3L0FUARD0PAERREVMNHUI8', 'yes'),
('A224TK7J4KA1LV', '2581SNQQ7Q67JWJLWQER92TQKJN9TD', 'yes'),
('A224TK7J4KA1LV', '2LISPW1INMHGHHMEF11BEBFMNXHH4I', 'yes'),
('A224TK7J4KA1LV', '2SIHFL42J1LYOZT2ELD0QGFKPVRN2Y', 'yes'),
('A224TK7J4KA1LV', '2ZRNZW6HEZ6OXV8RJ7Z6HGUMAJRZPE', 'yes'),
('A224TK7J4KA1LV', '214T6YTWX4H30T76GR09OI6IR91X2S', 'yes'),
('A224TK7J4KA1LV', '2T5HMVHVKCJ0KS2XJIA7HKQWEJ1646', 'yes'),
('A224TK7J4KA1LV', '2ALDHJHP4BDD37MTK5JXHX3GBNY4XW', 'yes'),
('A224TK7J4KA1LV', '2IDP9746OQ1S6H822KY0D1QKK5I2ME', 'yes'),
('A224TK7J4KA1LV', '2WIOHOVR14IX3FKGLG8EKWA65LO76O', 'yes'),
('A224TK7J4KA1LV', '2GYKPEIB9BAW4J4S14WV5UD3UBHX08', 'yes'),
('A224TK7J4KA1LV', '2WQ06UFBNFSVZL3AFNWS46NG9XLH3X', 'yes'),
('A224TK7J4KA1LV', '2L13NAB6BFMFYFGTU7STRFDGMA2UHF', 'yes'),
('A224TK7J4KA1LV', '2ALDHJHP4BDD37MTK5JXHX3GBN04XY', 'no'),
('A224TK7J4KA1LV', '2F4NCWYB49F9BJAC9FSJWNSTDZYOSK', 'yes'),
('A224TK7J4KA1LV', '2BXBNFSVGULFM0NN8KEG9FS3VGE7LX', 'yes'),
('A224TK7J4KA1LV', '20JLY58B727MJ9YAWV41EYFS3XXU9S', 'no'),
('A224TK7J4KA1LV', '2FKAOO2GMMEG7FYL4QD6QAA20W877N', 'yes'),
('A224TK7J4KA1LV', '20JLY58B727MJ9YAWV41EYFS3XU9U4', 'yes'),
('A224TK7J4KA1LV', '2A79RA7S5WM1P0GN0TY0RMHLI0JZUA', 'yes'),
('A224TK7J4KA1LV', '2AOYTWX4H3H282M8LN7IEIJRJNY4ZV', 'yes'),
('A2ZUENR4ZLC3MN', '2K5AB6BFMFFOHP0OD7AFLGESKIHJW8', 'yes')]
response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
self.assertEqual('OK', response['status'])
response = self.client.await_completion(self.client.get_estimated_workers_quality())
self.assertEqual('OK', response['status'])
def test_GH_88(self):
"""
Test for github issue https://github.com/ipeirotis/Troia-Server/issues/88
Thanks to PeccatorImpius for reporting
"""
categories = ['blank', 'confirm_closed', 'confirm_no', 'confirm_nonrestaurant', 'confirm_yes']
algorithm = "BDS" # TODO XXX FIXME currently it won't work on IDS
response = self.client.create(categories,
prioritycalculator="CostBased",
costMatrix=[ {"value":5.0, "to":"blank", "from":"blank"},
{"value":5.0, "to":"confirm_closed", "from":"blank"},
{"value":5.0, "to":"confirm_no", "from":"blank"},
{"value":5.0, "to":"confirm_nonrestaurant", "from":"blank"},
{"value":5.0, "to":"confirm_yes", "from":"blank"},
{"value":5.0, "to":"blank", "from":"confirm_closed"},
{"value":0.0, "to":"confirm_closed", "from":"confirm_closed"},
{"value":5.0, "to":"confirm_no", "from":"confirm_closed"},
{"value":5.0, "to":"confirm_nonrestaurant", "from":"confirm_closed"},
{"value":5.0, "to":"confirm_yes", "from":"confirm_closed"},
{"value":1.0, "to":"blank", "from":"confirm_no"},
{"value":1.0, "to":"confirm_closed", "from":"confirm_no"},
{"value":0.0, "to":"confirm_no", "from":"confirm_no"},
{"value":1.0, "to":"confirm_nonrestaurant", "from":"confirm_no"},
{"value":1.0, "to":"confirm_yes", "from":"confirm_no"},
{"value":5.0, "to":"blank", "from":"confirm_nonrestaurant"},
{"value":5.0, "to":"confirm_closed", "from":"confirm_nonrestaurant"},
{"value":5.0, "to":"confirm_no", "from":"confirm_nonrestaurant"},
{"value":0.0, "to":"confirm_nonrestaurant", "from":"confirm_nonrestaurant"},
{"value":5.0, "to":"confirm_yes", "from":"confirm_nonrestaurant"},
{"value":1.0, "to":"blank", "from":"confirm_yes"},
{"value":1.0, "to":"confirm_closed", "from":"confirm_yes"},
{"value":1.0, "to":"confirm_no", "from":"confirm_yes"},
{"value":1.0, "to":"confirm_nonrestaurant", "from":"confirm_yes"},
{"value":0.0, "to":"confirm_yes", "from":"confirm_yes"}],
epsilon=0.0001,
iterations=10,
algorithm=algorithm,
scheduler='NormalScheduler')
self.assertEqual('OK', response['status'])
assignedLabels = [
('A1R7CJMWXC79UO', '2OVZKPFE4EGD044XHZZIHNZWGWV7HC', 'confirm_yes'),
('A2VRQML8Q3XYP4', '2OVZKPFE4EGD044XHZZIHNZWGWV7HC', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2OVZKPFE4EGD044XHZZIHNZWGWV7HC', 'confirm_yes'),
('A3TXO6RKFIDFUV', '2Z3KH1Q6SVQ80ZQH28XLDOBVMXHL2G', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2Z3KH1Q6SVQ80ZQH28XLDOBVMXHL2G', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2Z3KH1Q6SVQ80ZQH28XLDOBVMXHL2G', 'confirm_yes'),
('A1FQYUBCBNTQX7', '2V6ZFYQS1CST5FXS3RJ4QC1E8S3KN3', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2V6ZFYQS1CST5FXS3RJ4QC1E8S3KN3', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2V6ZFYQS1CST5FXS3RJ4QC1E8S3KN3', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2MCFJ51Y7QEOI6GL4F3S1MOF76TBEL', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2MCFJ51Y7QEOI6GL4F3S1MOF76TBEL', 'confirm_yes'),
('A3TXO6RKFIDFUV', '2MCFJ51Y7QEOI6GL4F3S1MOF76TBEL', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2S4JTUHYW2GT8095J6WWU169874PK4', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2S4JTUHYW2GT8095J6WWU169874PK4', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2S4JTUHYW2GT8095J6WWU169874PK4', 'confirm_no'),
('A3PRF4IIM2IQN2', '2S3YHVI44OS23CC3P15Z0BAPFQI4YO', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2S3YHVI44OS23CC3P15Z0BAPFQI4YO', 'confirm_closed'),
('AH5ZIMHL7TNWC', '2S3YHVI44OS23CC3P15Z0BAPFQI4YO', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2KXDEY5COW0LZIHYKQIFTC6VX9G4VF', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2KXDEY5COW0LZIHYKQIFTC6VX9G4VF', 'confirm_yes'),
('A2KET1HL1COET5', '2KXDEY5COW0LZIHYKQIFTC6VX9G4VF', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2FT49F9SSSHXKS1JZ6K5P5F8TJ4TX2', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2FT49F9SSSHXKS1JZ6K5P5F8TJ4TX2', 'confirm_no'),
('A3TXO6RKFIDFUV', '2FT49F9SSSHXKS1JZ6K5P5F8TJ4TX2', 'confirm_no'),
('A3PRF4IIM2IQN2', '27G2RCJK63ZVXUZMCYLIIQ9JVK87WC', 'confirm_yes'),
('A19BL9GZGXFWFT', '27G2RCJK63ZVXUZMCYLIIQ9JVK87WC', 'confirm_yes'),
('AH5ZIMHL7TNWC', '27G2RCJK63ZVXUZMCYLIIQ9JVK87WC', 'confirm_no'),
('AH5ZIMHL7TNWC', '2JODG67D1X3VJ62VO2B2RE1MH40HAC', 'confirm_no'),
('A31X3JCHS0BPFJ', '2JODG67D1X3VJ62VO2B2RE1MH40HAC', 'confirm_yes'),
('A2VRQML8Q3XYP4', '2JODG67D1X3VJ62VO2B2RE1MH40HAC', 'confirm_yes'),
('A31X3JCHS0BPFJ', '26ZIT3HVWAVK1XKIV4T1F2Z9J5XEMD', 'confirm_closed'),
('A3TXO6RKFIDFUV', '26ZIT3HVWAVK1XKIV4T1F2Z9J5XEMD', 'confirm_closed'),
('AH5ZIMHL7TNWC', '26ZIT3HVWAVK1XKIV4T1F2Z9J5XEMD', 'confirm_nonrestaurant'),
('A3PRF4IIM2IQN2', '20N1Y7QEOZFY9JJ747DONXRD2JYEH4', 'confirm_yes'),
('A31X3JCHS0BPFJ', '20N1Y7QEOZFY9JJ747DONXRD2JYEH4', 'confirm_yes'),
('A2KET1HL1COET5', '20N1Y7QEOZFY9JJ747DONXRD2JYEH4', 'confirm_yes'),
('A3PRF4IIM2IQN2', '241KM05BMJTU0PEXS7G9ZA7SFB2EJ9', 'confirm_yes'),
('A2L2ZXRHK3SNOP', '241KM05BMJTU0PEXS7G9ZA7SFB2EJ9', 'confirm_yes'),
('AH5ZIMHL7TNWC', '241KM05BMJTU0PEXS7G9ZA7SFB2EJ9', 'confirm_no'),
('A3PRF4IIM2IQN2', '2YDWAVKI62NJ9TJ2ED09YH6BU0SRJZ', 'confirm_yes'),
('A19BL9GZGXFWFT', '2YDWAVKI62NJ9TJ2ED09YH6BU0SRJZ', 'confirm_closed'),
('AH5ZIMHL7TNWC', '2YDWAVKI62NJ9TJ2ED09YH6BU0SRJZ', 'confirm_no'),
('A3PRF4IIM2IQN2', '2JV21O3W5XH02G7NUGBYMPLSZMLHB7', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2JV21O3W5XH02G7NUGBYMPLSZMLHB7', 'confirm_yes'),
('A2L2ZXRHK3SNOP', '2JV21O3W5XH02G7NUGBYMPLSZMLHB7', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2JS1QP6AUC26W7O2PFO330FKAR5811', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2JS1QP6AUC26W7O2PFO330FKAR5811', 'confirm_yes'),
('A1I3CXC17NIRWB', '2JS1QP6AUC26W7O2PFO330FKAR5811', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2CIFK0COK2JEKDPKWY0LZW6O9PSRKY', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2CIFK0COK2JEKDPKWY0LZW6O9PSRKY', 'confirm_yes'),
('A2KET1HL1COET5', '2CIFK0COK2JEKDPKWY0LZW6O9PSRKY', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2IJK274J79KQFXJ3ZIXU5FCDEKDHFP', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2IJK274J79KQFXJ3ZIXU5FCDEKDHFP', 'confirm_no'),
('A2L2ZXRHK3SNOP', '2IJK274J79KQFXJ3ZIXU5FCDEKDHFP', 'confirm_yes'),
('A3TXO6RKFIDFUV', '234EGOOGQSCMP9S5E65I2UU0A36EEO', 'confirm_closed'),
('A3PRF4IIM2IQN2', '234EGOOGQSCMP9S5E65I2UU0A36EEO', 'confirm_closed'),
('A31X3JCHS0BPFJ', '234EGOOGQSCMP9S5E65I2UU0A36EEO', 'confirm_closed'),
('A3TXO6RKFIDFUV', '23QJIA0RYNJ9LE1FYEWBUJTURDD27P', 'confirm_yes'),
('A3PRF4IIM2IQN2', '23QJIA0RYNJ9LE1FYEWBUJTURDD27P', 'confirm_yes'),
('A31X3JCHS0BPFJ', '23QJIA0RYNJ9LE1FYEWBUJTURDD27P', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2S5VP3TVOK5UYANWALHEWZFY07I52D', 'confirm_no'),
('A2L2ZXRHK3SNOP', '2S5VP3TVOK5UYANWALHEWZFY07I52D', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2S5VP3TVOK5UYANWALHEWZFY07I52D', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2ETAT2D5NQYNO6LLCX751YMNMBF84E', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2ETAT2D5NQYNO6LLCX751YMNMBF84E', 'confirm_no'),
('A2L2ZXRHK3SNOP', '2ETAT2D5NQYNO6LLCX751YMNMBF84E', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2Z2MN9U8P9Y3RKER9WUS2YIMAY9EVU', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2Z2MN9U8P9Y3RKER9WUS2YIMAY9EVU', 'confirm_no'),
('A1I3CXC17NIRWB', '2Z2MN9U8P9Y3RKER9WUS2YIMAY9EVU', 'confirm_yes'),
('ABT7QTMIYXYO0', '2GXYQS1CSTMOYO984I9C9EYDXCZPM9', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2GXYQS1CSTMOYO984I9C9EYDXCZPM9', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2GXYQS1CSTMOYO984I9C9EYDXCZPM9', 'confirm_no'),
('AH5ZIMHL7TNWC', '2BLK4F0OHOVRKV0SW2TLH2HEMBR23K', 'blank'),
('A3PRF4IIM2IQN2', '2BLK4F0OHOVRKV0SW2TLH2HEMBR23K', 'confirm_yes'),
('A2L2ZXRHK3SNOP', '2BLK4F0OHOVRKV0SW2TLH2HEMBR23K', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2X6OGQSCM6IATTA9U8LU80OQUZQHHN', 'confirm_yes'),
('A2KET1HL1COET5', '2X6OGQSCM6IATTA9U8LU80OQUZQHHN', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2X6OGQSCM6IATTA9U8LU80OQUZQHHN', 'confirm_yes'),
('A31X3JCHS0BPFJ', '29UWY2AOO2GM55YJ0UHSKM6IKPI442', 'confirm_yes'),
('A3PRF4IIM2IQN2', '29UWY2AOO2GM55YJ0UHSKM6IKPI442', 'confirm_yes'),
('AH5ZIMHL7TNWC', '29UWY2AOO2GM55YJ0UHSKM6IKPI442', 'confirm_yes'),
('A31X3JCHS0BPFJ', '2DJVP9746OQ1BE8LJ4X7851QUR3L1Z', 'confirm_yes'),
('ABT7QTMIYXYO0', '2DJVP9746OQ1BE8LJ4X7851QUR3L1Z', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2DJVP9746OQ1BE8LJ4X7851QUR3L1Z', 'confirm_yes'),
('A31X3JCHS0BPFJ', '25XXRDS4IC1EH45SVTD19A2I5DKWZS', 'confirm_yes'),
('A3PRF4IIM2IQN2', '25XXRDS4IC1EH45SVTD19A2I5DKWZS', 'confirm_no'),
('AH5ZIMHL7TNWC', '25XXRDS4IC1EH45SVTD19A2I5DKWZS', 'confirm_no'),
('A2VRQML8Q3XYP4', '2DABRI8IUB2YD0QET6KLJ3L0PQZH56', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2DABRI8IUB2YD0QET6KLJ3L0PQZH56', 'confirm_no'),
('A3PRF4IIM2IQN2', '2DABRI8IUB2YD0QET6KLJ3L0PQZH56', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2TOSK9RJ85OZ1QWCYO1PUOOQFTCU60', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2TOSK9RJ85OZ1QWCYO1PUOOQFTCU60', 'confirm_yes'),
('A2L2ZXRHK3SNOP', '2TOSK9RJ85OZ1QWCYO1PUOOQFTCU60', 'confirm_yes'),
('A3PRF4IIM2IQN2', '2IBHV8ER9Y8T6B0HB6D5SFHSC334RM', 'confirm_yes'),
('ABT7QTMIYXYO0', '2IBHV8ER9Y8T6B0HB6D5SFHSC334RM', 'confirm_yes'),
('AH5ZIMHL7TNWC', '2IBHV8ER9Y8T6B0HB6D5SFHSC334RM', 'confirm_no'),
('A3TXO6RKFIDFUV', '294EZZ2MIKMNSLQKLCU81WWXSI97O0', 'confirm_yes'),
('A19BL9GZGXFWFT', '294EZZ2MIKMNSLQKLCU81WWXSI97O0', 'confirm_yes'),
('A1I3CXC17NIRWB', '294EZZ2MIKMNSLQKLCU81WWXSI97O0', 'confirm_yes'),
('A2VRQML8Q3XYP4', '2AOYTWX4H3H282M8LN7IEIJRLKM4ZF', 'confirm_yes')]
response = self.client.await_completion(self.client.post_assigned_labels(assignedLabels))
self.assertEqual('OK', response['status'])
if algorithm == "BDS":
self.client.await_completion(self.client.post_compute())
response = self.client.await_completion(self.client.get_estimated_workers_quality())
self.assertEqual('OK', response['status'])
for w in response['result']:
val = float(w['value'])
self.assertFalse(math.isnan(val))
