def testGetPendingWorkItems(self):
workerDataDB = initializeWorkerDataDB(self.testName)
workerData = WorkerData(None, {
"request_id": "foo1",
"worker_id": "abcd",
"fib_id": 3
})
workerDataDB.addWorkerData(workerData)
workerData = WorkerData(None, {
"request_id": "foo2",
"worker_id": "efgh",
"fib_id": 5
})
workerDataDB.addWorkerData(workerData)
timestamp = nowInSeconds()
workerDatas = workerDataDB.getWorkItems(isPending=True)
for workerData in workerDatas:
self.assertTrue(workerData.fibValue == -1)
self.assertTrue(workerData.finishedDate == None)
workerDatas = workerDataDB.getWorkItems(isPending=False)
self.assertTrue(len(workerDatas) == 0)
def testGetCompletedWorkItems(self):
workerDataDB = initializeWorkerDataDB(self.testName)
startDate = nowInSeconds()- 5;
workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3,"startedDate":startDate})
workerDataDB.addWorkerData(workerData)
workerData.fibValue = 3
workerDataDB.updateWorkerData(workerData)
workerData = WorkerData(None, {"request_id":"foo2","worker_id":"efgh","fib_id":5,"startedDate":startDate})
workerDataDB.addWorkerData(workerData)
workerData.fibValue = 8
workerDataDB.updateWorkerData(workerData)
workerDatas = workerDataDB.getWorkItems(isPending = True)
self.assertTrue(len(workerDatas) == 0)
workerDatas = workerDataDB.getWorkItems(isPending = False)
for workerData in workerDatas:
self.assertTrue(workerData.fibValue != -1)
self.assertTrue(workerData.finishedDate != None)
workerDatas = workerDataDB.getWorkItems(isPending = True)
self.assertTrue(len(workerDatas) == 0)
def processMessage(message):
'''
this is the handler function passed to getAndProcessMessages
'''
dataMap = {}
dataMap['worker_id'] = workerId
dataMap['request_id'] = message.requestId
dataMap['fib_id'] = message.messageKey
dataMap['fib_value'] = -1
dataMap['started_date'] = message.startedDate
workerData = WorkerData(body=dataMap)
log.debug(
"worker %s starting Fibonnaci on %d at %s" %
(workerId, workerData.fibId, prettyPrintTime(nowInSeconds())))
addedWorkerData = workerDataDB.getWorkerData(
requestId=workerData.requestId)
if addedWorkerData != None:
# if this is already in progress with another worker
if addedWorkerData.workerId == workerId:
log.error(
"expecting that this data was being worked on by a different worker, not %s"
% workerId)
workerData.retryCount = addedWorkerData.retryCount + 1
workerDataDB.updateWorkerData(workerData)
else:
addedWorkerData = workerDataDB.addWorkerData(workerData)
log.debug("starting fib(%d)" % workerData.fibId)
fibValue = F(message.messageKey)
addedWorkerData.fibValue = fibValue
log.debug("completed fib(%d), value = %d" %
(workerData.fibId, fibValue))
addedWorkerData.finishedDate = nowInSeconds()
workerDataDB.updateWorkerData(addedWorkerData)
log.debug(
"worker %s finished Fibonnaci on %d, calculated value = %d, at %s"
% (workerId, addedWorkerData.fibId, addedWorkerData.fibValue,
prettyPrintTime(addedWorkerData.finishedDate)))
# now send result back for result processing.
resultsMessageQueue.sendMessage(
WorkResultMessage(requestId=addedWorkerData.requestId,
messageKey=addedWorkerData.fibId,
messageValue=addedWorkerData.fibValue,
startedDate=addedWorkerData.startedDate,
finishedDate=addedWorkerData.finishedDate))
def testUpdateWorkerData(self):
workerDataDB = initializeWorkerDataDB(self.testName)
lastCheckinDate = nowInSeconds()
lastCheckinDate = lastCheckinDate-5
workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3,"started_date":nowInSeconds()-5})
newWorkerData = workerDataDB.addWorkerData(workerData)
workerData.fibValue=3
workerDataDB.updateWorkerData(workerData)
checkWorkerData = workerDataDB.getWorkerData(workerData.workerId)
self.assertTrue(checkWorkerData.finishedDate != None)
def testUpdateWorkerData(self):
workerDataDB = initializeWorkerDataDB(self.testName)
lastCheckinDate = nowInSeconds()
lastCheckinDate = lastCheckinDate - 5
workerData = WorkerData(
None, {
"request_id": "foo1",
"worker_id": "abcd",
"fib_id": 3,
"started_date": nowInSeconds() - 5
})
newWorkerData = workerDataDB.addWorkerData(workerData)
workerData.fibValue = 3
workerDataDB.updateWorkerData(workerData)
checkWorkerData = workerDataDB.getWorkerData(workerData.workerId)
self.assertTrue(checkWorkerData.finishedDate != None)
def test1AddWorkerDataAtStart(self):
workerDataDB = initializeWorkerDataDB(self.testName)
workerData = WorkerData(None, {
"request_id": "foo1",
"worker_id": "abcd",
"fib_id": 3
})
newWorkerData = workerDataDB.addWorkerData(workerData)
assert newWorkerData.workerId > 0
def processMessage(message):
'''
this is the handler function passed to getAndProcessMessages
'''
dataMap = {}
dataMap['worker_id'] = workerId
dataMap['request_id'] = message.requestId
dataMap['fib_id'] = message.messageKey
dataMap['fib_value'] = -1
dataMap['started_date'] = message.startedDate
workerData = WorkerData(body=dataMap)
log.debug("worker %s starting Fibonnaci on %d at %s"%(workerId,workerData.fibId,prettyPrintTime(nowInSeconds())) )
addedWorkerData = workerDataDB.getWorkerData(requestId = workerData.requestId)
if addedWorkerData != None:
# if this is already in progress with another worker
if addedWorkerData.workerId == workerId:
log.error("expecting that this data was being worked on by a different worker, not %s"%workerId)
workerData.retryCount = addedWorkerData.retryCount + 1
workerDataDB.updateWorkerData(workerData)
else:
addedWorkerData = workerDataDB.addWorkerData(workerData)
log.debug("starting fib(%d)"%workerData.fibId)
fibValue = F(message.messageKey)
addedWorkerData.fibValue = fibValue
log.debug("completed fib(%d), value = %d"%(workerData.fibId,fibValue))
addedWorkerData.finishedDate = nowInSeconds()
workerDataDB.updateWorkerData(addedWorkerData)
log.debug("worker %s finished Fibonnaci on %d, calculated value = %d, at %s"%(workerId,addedWorkerData.fibId,addedWorkerData.fibValue,prettyPrintTime(addedWorkerData.finishedDate)) )
# now send result back for result processing.
resultsMessageQueue.sendMessage(WorkResultMessage(requestId = addedWorkerData.requestId, messageKey = addedWorkerData.fibId, messageValue = addedWorkerData.fibValue,startedDate = addedWorkerData.startedDate,finishedDate = addedWorkerData.finishedDate))
def testUpdateRetryCount(self):
workerDataDB = initializeWorkerDataDB(self.testName)
workerData = WorkerData(None, {
"request_id": "foo1",
"worker_id": "abcd",
"fib_id": 3
})
workerDataDB.addWorkerData(workerData)
newRetryCount = workerData.retryCount = workerData.retryCount + 1
workerDataDB.updateWorkerData(workerData)
updatedWorkerData = workerDataDB.getWorkerData(workerId="abcd")
self.assertTrue(updatedWorkerData.retryCount == newRetryCount)
def test2GetWorkerData(self):
workerDataDB = initializeWorkerDataDB(self.testName)
workerData = WorkerData(None, {
"request_id": "foo1",
"worker_id": "abcd",
"fib_id": 3
})
newWorkerData = workerDataDB.addWorkerData(workerData)
checkWorkerData = workerDataDB.getWorkerData(
workerId=newWorkerData.workerId)
self.assertTrue(checkWorkerData.requestId == workerData.requestId)
self.assertTrue(checkWorkerData.workerId == workerData.workerId)
self.assertTrue(checkWorkerData.fibId == workerData.fibId)
checkWorkerData = workerDataDB.getWorkerData(
requestId=newWorkerData.requestId)
self.assertTrue(checkWorkerData.requestId == workerData.requestId)
self.assertTrue(checkWorkerData.workerId == workerData.workerId)
self.assertTrue(checkWorkerData.fibId == workerData.fibId)
def testGetCompletedWorkItems(self):
workerDataDB = initializeWorkerDataDB(self.testName)
startDate = nowInSeconds() - 5
workerData = WorkerData(
None, {
"request_id": "foo1",
"worker_id": "abcd",
"fib_id": 3,
"startedDate": startDate
})
workerDataDB.addWorkerData(workerData)
workerData.fibValue = 3
workerDataDB.updateWorkerData(workerData)
workerData = WorkerData(
None, {
"request_id": "foo2",
"worker_id": "efgh",
"fib_id": 5,
"startedDate": startDate
})
workerDataDB.addWorkerData(workerData)
workerData.fibValue = 8
workerDataDB.updateWorkerData(workerData)
workerDatas = workerDataDB.getWorkItems(isPending=True)
self.assertTrue(len(workerDatas) == 0)
workerDatas = workerDataDB.getWorkItems(isPending=False)
for workerData in workerDatas:
self.assertTrue(workerData.fibValue != -1)
self.assertTrue(workerData.finishedDate != None)
workerDatas = workerDataDB.getWorkItems(isPending=True)
self.assertTrue(len(workerDatas) == 0)
