def testGetPendingWorkItems(self): workerDataDB = initializeWorkerDataDB(self.testName) workerData = WorkerData(None, { "request_id": "foo1", "worker_id": "abcd", "fib_id": 3 }) workerDataDB.addWorkerData(workerData) workerData = WorkerData(None, { "request_id": "foo2", "worker_id": "efgh", "fib_id": 5 }) workerDataDB.addWorkerData(workerData) timestamp = nowInSeconds() workerDatas = workerDataDB.getWorkItems(isPending=True) for workerData in workerDatas: self.assertTrue(workerData.fibValue == -1) self.assertTrue(workerData.finishedDate == None) workerDatas = workerDataDB.getWorkItems(isPending=False) self.assertTrue(len(workerDatas) == 0)
def testGetCompletedWorkItems(self): workerDataDB = initializeWorkerDataDB(self.testName) startDate = nowInSeconds()- 5; workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3,"startedDate":startDate}) workerDataDB.addWorkerData(workerData) workerData.fibValue = 3 workerDataDB.updateWorkerData(workerData) workerData = WorkerData(None, {"request_id":"foo2","worker_id":"efgh","fib_id":5,"startedDate":startDate}) workerDataDB.addWorkerData(workerData) workerData.fibValue = 8 workerDataDB.updateWorkerData(workerData) workerDatas = workerDataDB.getWorkItems(isPending = True) self.assertTrue(len(workerDatas) == 0) workerDatas = workerDataDB.getWorkItems(isPending = False) for workerData in workerDatas: self.assertTrue(workerData.fibValue != -1) self.assertTrue(workerData.finishedDate != None) workerDatas = workerDataDB.getWorkItems(isPending = True) self.assertTrue(len(workerDatas) == 0)
def processMessage(message): ''' this is the handler function passed to getAndProcessMessages ''' dataMap = {} dataMap['worker_id'] = workerId dataMap['request_id'] = message.requestId dataMap['fib_id'] = message.messageKey dataMap['fib_value'] = -1 dataMap['started_date'] = message.startedDate workerData = WorkerData(body=dataMap) log.debug( "worker %s starting Fibonnaci on %d at %s" % (workerId, workerData.fibId, prettyPrintTime(nowInSeconds()))) addedWorkerData = workerDataDB.getWorkerData( requestId=workerData.requestId) if addedWorkerData != None: # if this is already in progress with another worker if addedWorkerData.workerId == workerId: log.error( "expecting that this data was being worked on by a different worker, not %s" % workerId) workerData.retryCount = addedWorkerData.retryCount + 1 workerDataDB.updateWorkerData(workerData) else: addedWorkerData = workerDataDB.addWorkerData(workerData) log.debug("starting fib(%d)" % workerData.fibId) fibValue = F(message.messageKey) addedWorkerData.fibValue = fibValue log.debug("completed fib(%d), value = %d" % (workerData.fibId, fibValue)) addedWorkerData.finishedDate = nowInSeconds() workerDataDB.updateWorkerData(addedWorkerData) log.debug( "worker %s finished Fibonnaci on %d, calculated value = %d, at %s" % (workerId, addedWorkerData.fibId, addedWorkerData.fibValue, prettyPrintTime(addedWorkerData.finishedDate))) # now send result back for result processing. resultsMessageQueue.sendMessage( WorkResultMessage(requestId=addedWorkerData.requestId, messageKey=addedWorkerData.fibId, messageValue=addedWorkerData.fibValue, startedDate=addedWorkerData.startedDate, finishedDate=addedWorkerData.finishedDate))
def testUpdateWorkerData(self): workerDataDB = initializeWorkerDataDB(self.testName) lastCheckinDate = nowInSeconds() lastCheckinDate = lastCheckinDate-5 workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3,"started_date":nowInSeconds()-5}) newWorkerData = workerDataDB.addWorkerData(workerData) workerData.fibValue=3 workerDataDB.updateWorkerData(workerData) checkWorkerData = workerDataDB.getWorkerData(workerData.workerId) self.assertTrue(checkWorkerData.finishedDate != None)
def testUpdateWorkerData(self): workerDataDB = initializeWorkerDataDB(self.testName) lastCheckinDate = nowInSeconds() lastCheckinDate = lastCheckinDate - 5 workerData = WorkerData( None, { "request_id": "foo1", "worker_id": "abcd", "fib_id": 3, "started_date": nowInSeconds() - 5 }) newWorkerData = workerDataDB.addWorkerData(workerData) workerData.fibValue = 3 workerDataDB.updateWorkerData(workerData) checkWorkerData = workerDataDB.getWorkerData(workerData.workerId) self.assertTrue(checkWorkerData.finishedDate != None)
def test1AddWorkerDataAtStart(self): workerDataDB = initializeWorkerDataDB(self.testName) workerData = WorkerData(None, { "request_id": "foo1", "worker_id": "abcd", "fib_id": 3 }) newWorkerData = workerDataDB.addWorkerData(workerData) assert newWorkerData.workerId > 0
def processMessage(message): ''' this is the handler function passed to getAndProcessMessages ''' dataMap = {} dataMap['worker_id'] = workerId dataMap['request_id'] = message.requestId dataMap['fib_id'] = message.messageKey dataMap['fib_value'] = -1 dataMap['started_date'] = message.startedDate workerData = WorkerData(body=dataMap) log.debug("worker %s starting Fibonnaci on %d at %s"%(workerId,workerData.fibId,prettyPrintTime(nowInSeconds())) ) addedWorkerData = workerDataDB.getWorkerData(requestId = workerData.requestId) if addedWorkerData != None: # if this is already in progress with another worker if addedWorkerData.workerId == workerId: log.error("expecting that this data was being worked on by a different worker, not %s"%workerId) workerData.retryCount = addedWorkerData.retryCount + 1 workerDataDB.updateWorkerData(workerData) else: addedWorkerData = workerDataDB.addWorkerData(workerData) log.debug("starting fib(%d)"%workerData.fibId) fibValue = F(message.messageKey) addedWorkerData.fibValue = fibValue log.debug("completed fib(%d), value = %d"%(workerData.fibId,fibValue)) addedWorkerData.finishedDate = nowInSeconds() workerDataDB.updateWorkerData(addedWorkerData) log.debug("worker %s finished Fibonnaci on %d, calculated value = %d, at %s"%(workerId,addedWorkerData.fibId,addedWorkerData.fibValue,prettyPrintTime(addedWorkerData.finishedDate)) ) # now send result back for result processing. resultsMessageQueue.sendMessage(WorkResultMessage(requestId = addedWorkerData.requestId, messageKey = addedWorkerData.fibId, messageValue = addedWorkerData.fibValue,startedDate = addedWorkerData.startedDate,finishedDate = addedWorkerData.finishedDate))
def testUpdateRetryCount(self): workerDataDB = initializeWorkerDataDB(self.testName) workerData = WorkerData(None, { "request_id": "foo1", "worker_id": "abcd", "fib_id": 3 }) workerDataDB.addWorkerData(workerData) newRetryCount = workerData.retryCount = workerData.retryCount + 1 workerDataDB.updateWorkerData(workerData) updatedWorkerData = workerDataDB.getWorkerData(workerId="abcd") self.assertTrue(updatedWorkerData.retryCount == newRetryCount)
def test2GetWorkerData(self): workerDataDB = initializeWorkerDataDB(self.testName) workerData = WorkerData(None, { "request_id": "foo1", "worker_id": "abcd", "fib_id": 3 }) newWorkerData = workerDataDB.addWorkerData(workerData) checkWorkerData = workerDataDB.getWorkerData( workerId=newWorkerData.workerId) self.assertTrue(checkWorkerData.requestId == workerData.requestId) self.assertTrue(checkWorkerData.workerId == workerData.workerId) self.assertTrue(checkWorkerData.fibId == workerData.fibId) checkWorkerData = workerDataDB.getWorkerData( requestId=newWorkerData.requestId) self.assertTrue(checkWorkerData.requestId == workerData.requestId) self.assertTrue(checkWorkerData.workerId == workerData.workerId) self.assertTrue(checkWorkerData.fibId == workerData.fibId)
def testGetCompletedWorkItems(self): workerDataDB = initializeWorkerDataDB(self.testName) startDate = nowInSeconds() - 5 workerData = WorkerData( None, { "request_id": "foo1", "worker_id": "abcd", "fib_id": 3, "startedDate": startDate }) workerDataDB.addWorkerData(workerData) workerData.fibValue = 3 workerDataDB.updateWorkerData(workerData) workerData = WorkerData( None, { "request_id": "foo2", "worker_id": "efgh", "fib_id": 5, "startedDate": startDate }) workerDataDB.addWorkerData(workerData) workerData.fibValue = 8 workerDataDB.updateWorkerData(workerData) workerDatas = workerDataDB.getWorkItems(isPending=True) self.assertTrue(len(workerDatas) == 0) workerDatas = workerDataDB.getWorkItems(isPending=False) for workerData in workerDatas: self.assertTrue(workerData.fibValue != -1) self.assertTrue(workerData.finishedDate != None) workerDatas = workerDataDB.getWorkItems(isPending=True) self.assertTrue(len(workerDatas) == 0)