def __init__(self, requestId = None, messageKey = None, body = None): messageLogger.debug("initializing from JSON") if body != None: if body.has_key('requestId') == True: self.requestId = body['requestId'] else: messageLogger.error("invalid JSON format, requestId not found") raise 'invalid format' if body.has_key('messageKey') == True: self.messageKey = body['messageKey'] else: messageLogger.error("invalid JSON format, messageKey not found") raise 'invalid format' # created is optional. if body.has_key('startedDate'): self.starteDate = body['startedDate'] else: self.startedDate = nowInSeconds() else: self.requestId = requestId self.messageKey = messageKey self.createdDate = nowInSeconds()
def updateWorkerData(self, workerData): """ updates a specified workerData, setting workerStatus and lastCheckinDate values ONLY. (those are the things that can change) """ try: db = self.connectToDB() cur = db.cursor() query = None if (workerData.fibValue > -1): self.log.debug( "update workerdata set fib_value = %d, finished_date = %d, retry_count = %d where request_id='%s'" % (workerData.fibValue, nowInSeconds(), workerData.retryCount, workerData.requestId)) query = "update workerdata set fib_value = %d, finished_date = %d, retry_count = %d where request_id='%s'" % ( workerData.fibValue, nowInSeconds(), workerData.retryCount, workerData.requestId) else: self.log.debug( "update workerdata set fib_value = %d, started_date = %d, worker_id = '%s', retry_count = %d where request_id='%s'" % (workerData.fibValue, workerData.startedDate, workerData.workerId, workerData.retryCount, workerData.requestId)) query = "update workerdata set retry_count = %d where request_id='%s'" % ( workerData.retryCount, workerData.requestId) cur.execute(query) db.commit() self.disconnectFromDB(db) except MySQLdb.Error as e: self.log.error(str(e)) self.handleMySQLException(e)
def addWorkerData(self, workerData): """ inserts a workerData into the database and timestamps it for readability """ try: db = self.connectToDB() cur = db.cursor() self.log.debug( "adding worker entry into database with request_id '%s', worker_id = '%s', fib_id=%d, startedDate = %s" % (workerData.requestId, workerData.workerId, workerData.fibId, nowInSeconds())) query = "insert into workerdata(request_id, worker_id, fib_id,started_date) values('%s','%s',%d,%d)" % ( workerData.requestId, workerData.workerId, workerData.fibId, nowInSeconds()) cur.execute(query) db.commit() self.disconnectFromDB(db) return workerData except MySQLdb.Error as e: self.log.error(str(e)) self.handleMySQLException(e)
def __init__(self, requestId=None, messageKey=None, body=None): messageLogger.debug("initializing from JSON") if body != None: if body.has_key('requestId') == True: self.requestId = body['requestId'] else: messageLogger.error("invalid JSON format, requestId not found") raise 'invalid format' if body.has_key('messageKey') == True: self.messageKey = body['messageKey'] else: messageLogger.error( "invalid JSON format, messageKey not found") raise 'invalid format' # created is optional. if body.has_key('startedDate'): self.starteDate = body['startedDate'] else: self.startedDate = nowInSeconds() else: self.requestId = requestId self.messageKey = messageKey self.createdDate = nowInSeconds()
def test12Add_UpdateRequest_FetchAllCompleteTasks(self): fibDataDB = initializeFibDataDB(self.testName) finishedDate = nowInSeconds() startedDate = finishedDate - 5 fd = FibDataRequest( None, { "request_id": "foo1", "fib_id": 3, "fib_value": 3, "started_date": startedDate, "finished_date": finishedDate }) fibDataDB.addRequest(fd) fdArr = fibDataDB.getRequests(isPending=False) self.assertTrue(len(fdArr) == 1) startedDate = nowInSeconds() fd = FibDataRequest( None, { "request_id": "foo2", "fib_id": 3, "fib_value": 3, "started_date": startedDate }) fibDataDB.addRequest(fd) fdArr = fibDataDB.getRequests(isPending=False) self.assertTrue(len(fdArr) == 1)
def processMessage(message): ''' this is the handler function passed to getAndProcessMessages ''' dataMap = {} dataMap['worker_id'] = workerId dataMap['request_id'] = message.requestId dataMap['fib_id'] = message.messageKey dataMap['fib_value'] = -1 dataMap['started_date'] = message.startedDate workerData = WorkerData(body=dataMap) log.debug( "worker %s starting Fibonnaci on %d at %s" % (workerId, workerData.fibId, prettyPrintTime(nowInSeconds()))) addedWorkerData = workerDataDB.getWorkerData( requestId=workerData.requestId) if addedWorkerData != None: # if this is already in progress with another worker if addedWorkerData.workerId == workerId: log.error( "expecting that this data was being worked on by a different worker, not %s" % workerId) workerData.retryCount = addedWorkerData.retryCount + 1 workerDataDB.updateWorkerData(workerData) else: addedWorkerData = workerDataDB.addWorkerData(workerData) log.debug("starting fib(%d)" % workerData.fibId) fibValue = F(message.messageKey) addedWorkerData.fibValue = fibValue log.debug("completed fib(%d), value = %d" % (workerData.fibId, fibValue)) addedWorkerData.finishedDate = nowInSeconds() workerDataDB.updateWorkerData(addedWorkerData) log.debug( "worker %s finished Fibonnaci on %d, calculated value = %d, at %s" % (workerId, addedWorkerData.fibId, addedWorkerData.fibValue, prettyPrintTime(addedWorkerData.finishedDate))) # now send result back for result processing. resultsMessageQueue.sendMessage( WorkResultMessage(requestId=addedWorkerData.requestId, messageKey=addedWorkerData.fibId, messageValue=addedWorkerData.fibValue, startedDate=addedWorkerData.startedDate, finishedDate=addedWorkerData.finishedDate))
def testGetPendingWorkItems(self): workerDataDB = initializeWorkerDataDB(self.testName) workerData = WorkerData(None, { "request_id": "foo1", "worker_id": "abcd", "fib_id": 3 }) workerDataDB.addWorkerData(workerData) workerData = WorkerData(None, { "request_id": "foo2", "worker_id": "efgh", "fib_id": 5 }) workerDataDB.addWorkerData(workerData) timestamp = nowInSeconds() workerDatas = workerDataDB.getWorkItems(isPending=True) for workerData in workerDatas: self.assertTrue(workerData.fibValue == -1) self.assertTrue(workerData.finishedDate == None) workerDatas = workerDataDB.getWorkItems(isPending=False) self.assertTrue(len(workerDatas) == 0)
def __init__(self, workerData): self.workerData = workerData self.formattedStartDate = prettyPrintTime(workerData.startedDate) if workerData.finishedDate != None: self.formattedFinishDate = prettyPrintTime(workerData.finishedDate) else: self.runTime = nowInSeconds() - workerData.startedDate
def test3GetAllRequests(self): try: fibDataDB = initializeFibDataDB(self.testName) startedDate = nowInSeconds() fd = FibDataRequest( None, { "request_id": "foo2", "fib_id": 3, "fib_value": 3, "started_date": startedDate }) newFd = fibDataDB.addRequest(fd) fdArr = fibDataDB.getRequests(isPending=True) self.assertTrue(len(fdArr) > 0) self.assertTrue(len(fdArr) == 1) testFd = fdArr[0] self.assertTrue(testFd.fibId == newFd.fibId) fdArr2 = fibDataDB.getRequests(isPending=True, isDescending=False) self.assertTrue(len(fdArr2) == 1) testFd2 = fdArr2[0] self.assertTrue(testFd2.fibId == newFd.fibId) fdArr3 = fibDataDB.getRequests() self.assertTrue(len(fdArr3) == 0) except: e = sys.exc_info()[0] print e self.fail(e)
def test13SerializeDisplayData(self): try: fibDataDB = initializeFibDataDB(self.testName) startedDate = nowInSeconds() fd = FibDataRequest(None, {"request_id":"foo1","fib_id":3,"fib_value":3,"started_date": startedDate}) fd2 = FibDataRequest(None,{"request_id":"foo2","fib_id":4,"fib_value":5,"started_date": startedDate}) fibDataDB.addRequest(fd) fibDataDB.addRequest(fd2) fdArr = fibDataDB.getRequests(isPending = True) self.assertTrue(len(fdArr) == 2) displayDataArr = [] for request in fdArr: displayDataArr.append(DisplayData(request)) str = json.dumps(displayDataArr,cls=DataEncoder) print str except: e = sys.exc_info()[0] print e self.fail(e)
def addRequest(self,request): """ inserts a request into the database and timestamps it for readability """ try: db = self.connectToDB() cur = db.cursor() if request.startedDate == None: request.startedDate = nowInSeconds() if request.finishedDate == None: self.log.debug("adding request into database with request_id = %s, fib_id = %d, fib_value = %d and started_date = %d"%(request.requestId,request.fibId,request.fibValue,request.startedDate)) query = "insert into fibdata(request_id,fib_id,fib_value,started_date) values('%s', %d, %d,%d)"%( request.requestId,request.fibId, request.fibValue,request.startedDate) else: self.log.debug("adding request into database with request_id = %s, fib_id = %d, fib_value = %d, started_date = %d, finished_date = %d"%(request.requestId,request.fibId,request.fibValue,request.startedDate,request.finishedDate)) query = "insert into fibdata(request_id,fib_id,fib_value,started_date,finished_date) values('%s',%d, %d,%d,%d)"%(request.requestId,request.fibId, request.fibValue,request.startedDate,request.finishedDate) cur.execute(query) db.commit() self.disconnectFromDB(db) return request except MySQLdb.Error as e: self.log.error(str(e)) self.handleMySQLException(e)
def updateRequest(self, request): """ updates a specified request, setting workerId, fibValue and finished_time values ONLY. (those are the things that can change) """ # explicit fail if the record hasn't been added if request.requestId == -1: raise 'cannot update a request that has not been added already' try: db = self.connectToDB() cur = db.cursor() finishedDate = nowInSeconds() self.log.debug( "updating request with fib_id = %d and fib_value = %d and requestId = '%s'" % (request.fibId, request.fibValue, request.requestId)) query = "update fibdata set fib_value=%d,finished_date=%d where request_id = '%s'" % ( request.fibValue, finishedDate, request.requestId) cur.execute(query) db.commit() self.disconnectFromDB(db) except MySQLdb.Error as e: self.log.error(str(e)) self.handleMySQLException(e)
def testGetCompletedWorkItems(self): workerDataDB = initializeWorkerDataDB(self.testName) startDate = nowInSeconds()- 5; workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3,"startedDate":startDate}) workerDataDB.addWorkerData(workerData) workerData.fibValue = 3 workerDataDB.updateWorkerData(workerData) workerData = WorkerData(None, {"request_id":"foo2","worker_id":"efgh","fib_id":5,"startedDate":startDate}) workerDataDB.addWorkerData(workerData) workerData.fibValue = 8 workerDataDB.updateWorkerData(workerData) workerDatas = workerDataDB.getWorkItems(isPending = True) self.assertTrue(len(workerDatas) == 0) workerDatas = workerDataDB.getWorkItems(isPending = False) for workerData in workerDatas: self.assertTrue(workerData.fibValue != -1) self.assertTrue(workerData.finishedDate != None) workerDatas = workerDataDB.getWorkItems(isPending = True) self.assertTrue(len(workerDatas) == 0)
def test2AddRequest(self): try: fibDataDB = initializeFibDataDB(self.testName) startedDate = nowInSeconds() fd = FibDataRequest( None, { "request_id": "foo1", "fib_id": 3, "fib_value": 3, "started_date": startedDate }) newFd = fibDataDB.addRequest(fd) fetchedFd = fibDataDB.getRequest(newFd.requestId) self.assertTrue(fd.fibId == fetchedFd.fibId) self.assertTrue(fd.fibValue == fetchedFd.fibValue) self.assertTrue(fd.startedDate == fetchedFd.startedDate) except: e = sys.exc_info()[0] print e self.fail(e)
def test3GetAllRequests(self): try: fibDataDB = initializeFibDataDB(self.testName) startedDate = nowInSeconds() fd = FibDataRequest(None, {"request_id":"foo2","fib_id":3,"fib_value":3,"started_date": startedDate}) newFd = fibDataDB.addRequest(fd) fdArr = fibDataDB.getRequests(isPending = True) self.assertTrue(len(fdArr) > 0) self.assertTrue(len(fdArr) == 1) testFd = fdArr[0] self.assertTrue(testFd.fibId == newFd.fibId) fdArr2 = fibDataDB.getRequests(isPending = True,isDescending=False) self.assertTrue(len(fdArr2) == 1) testFd2 = fdArr2[0] self.assertTrue(testFd2.fibId == newFd.fibId) fdArr3 = fibDataDB.getRequests() self.assertTrue(len(fdArr3) == 0) except: e = sys.exc_info()[0] print e self.fail(e)
def __init__(self, row = None, body = None): # SELECT id,request_id, worker_id,fib_id,fib_value, started_date,finished_date if row != None: workerDataLogger.debug("initializing from database") self.requestId = row[0] self.workerId = row[1] self.fibId = row[2] self.fibValue = row[3] self.retryCount = row[4] self.startedDate = row[5] if len(row) == 7: self.finishedDate = row[6] else: self.finishedDate = None elif body != None: workerDataLogger.debug("initializing from JSON") if body.has_key('request_id') == True: self.requestId = body['request_id'] else: workerDataLogger.error("invalid JSON format, request_id not found") raise 'invalid format' if body.has_key('worker_id') == True: self.workerId = body['worker_id'] else: workerDataLogger.error("invalid JSON format, worker_id not found") raise 'invalid format' if body.has_key('fib_id') == True: self.fibId = body['fib_id'] else: workerDataLogger.error("invalid JSON format, fib_id not found") raise 'invalid format' if body.has_key('fib_value'): self.fibValue = body['fib_value'] else: self.fibValue = -1 if body.has_key("retry_count"): self.finishedDate = body['retry_count'] else: self.retryCount = 0 if body.has_key("started_date"): self.startedDate = body['started_date'] else: self.startedDate = nowInSeconds() if body.has_key("finished_date"): self.finishedDate = body['finished_date'] else: self.finishedDate = None
def __init__(self, row=None, body=None): # SELECT id,request_id, worker_id,fib_id,fib_value, started_date,finished_date if row != None: workerDataLogger.debug("initializing from database") self.requestId = row[0] self.workerId = row[1] self.fibId = row[2] self.fibValue = row[3] self.retryCount = row[4] self.startedDate = row[5] if len(row) == 7: self.finishedDate = row[6] else: self.finishedDate = None elif body != None: workerDataLogger.debug("initializing from JSON") if body.has_key('request_id') == True: self.requestId = body['request_id'] else: workerDataLogger.error( "invalid JSON format, request_id not found") raise 'invalid format' if body.has_key('worker_id') == True: self.workerId = body['worker_id'] else: workerDataLogger.error( "invalid JSON format, worker_id not found") raise 'invalid format' if body.has_key('fib_id') == True: self.fibId = body['fib_id'] else: workerDataLogger.error("invalid JSON format, fib_id not found") raise 'invalid format' if body.has_key('fib_value'): self.fibValue = body['fib_value'] else: self.fibValue = -1 if body.has_key("retry_count"): self.finishedDate = body['retry_count'] else: self.retryCount = 0 if body.has_key("started_date"): self.startedDate = body['started_date'] else: self.startedDate = nowInSeconds() if body.has_key("finished_date"): self.finishedDate = body['finished_date'] else: self.finishedDate = None
def test12Add_UpdateRequest_FetchAllCompleteTasks(self): fibDataDB = initializeFibDataDB(self.testName) finishedDate = nowInSeconds() startedDate = finishedDate - 5; fd = FibDataRequest(None, {"request_id":"foo1","fib_id":3,"fib_value":3,"started_date": startedDate,"finished_date":finishedDate}) fibDataDB.addRequest(fd) fdArr = fibDataDB.getRequests(isPending = False) self.assertTrue(len(fdArr) == 1) startedDate = nowInSeconds() fd = FibDataRequest(None, {"request_id":"foo2","fib_id":3,"fib_value":3,"started_date": startedDate}) fibDataDB.addRequest(fd) fdArr = fibDataDB.getRequests(isPending = False) self.assertTrue(len(fdArr) == 1)
def testUpdateWorkerData(self): workerDataDB = initializeWorkerDataDB(self.testName) lastCheckinDate = nowInSeconds() lastCheckinDate = lastCheckinDate - 5 workerData = WorkerData( None, { "request_id": "foo1", "worker_id": "abcd", "fib_id": 3, "started_date": nowInSeconds() - 5 }) newWorkerData = workerDataDB.addWorkerData(workerData) workerData.fibValue = 3 workerDataDB.updateWorkerData(workerData) checkWorkerData = workerDataDB.getWorkerData(workerData.workerId) self.assertTrue(checkWorkerData.finishedDate != None)
def test6AddRequestWithFinishedDate(self): try: fibDataDB = initializeFibDataDB(self.testName) fibData = FibDataRequest( None, { "request_id": "foo1", "fib_id": 3, "fib_value": 2, "started_date": nowInSeconds(), "finished_date": nowInSeconds() }) newFD = fibDataDB.addRequest(fibData) self.assertTrue(newFD.requestId != -1) self.assertTrue(newFD.fibId == fibData.fibId) self.assertTrue(newFD.fibValue == fibData.fibValue) self.assertTrue(newFD.startedDate == fibData.startedDate) self.assertTrue(newFD.finishedDate == fibData.finishedDate) except: e = sys.exc_info()[0] print e self.fail(e)
def test7Add_UpdateRequest(self): try: fibDataDB = initializeFibDataDB(self.testName) fibData = FibDataRequest( None, { "request_id": "foo1", "fib_id": 3, "fib_value": 2, "started_date": nowInSeconds(), "finished_date": nowInSeconds() }) newFD = fibDataDB.addRequest(fibData) finishedDate = nowInSeconds() + 5 fibData.finishedDate = finishedDate fibDataDB.updateRequest(fibData) except: e = sys.exc_info()[0] print e self.fail(e)
def testUpdateWorkerData(self): workerDataDB = initializeWorkerDataDB(self.testName) lastCheckinDate = nowInSeconds() lastCheckinDate = lastCheckinDate-5 workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3,"started_date":nowInSeconds()-5}) newWorkerData = workerDataDB.addWorkerData(workerData) workerData.fibValue=3 workerDataDB.updateWorkerData(workerData) checkWorkerData = workerDataDB.getWorkerData(workerData.workerId) self.assertTrue(checkWorkerData.finishedDate != None)
def test7Add_UpdateRequest(self): try: fibDataDB = initializeFibDataDB(self.testName) fibData = FibDataRequest(None,{"request_id":"foo1","fib_id":3, "fib_value":2, "started_date" :nowInSeconds(), "finished_date" : nowInSeconds()}) newFD = fibDataDB.addRequest(fibData) finishedDate = nowInSeconds()+5 fibData.finishedDate = finishedDate fibDataDB.updateRequest(fibData) except: e = sys.exc_info()[0] print e self.fail(e)
def test5InitializeFromJSON(self): try: dataMap = {} dataMap['request_id'] = "foo1" dataMap['fib_id'] = 3 dataMap['fib_value'] = 3 dataMap['started_date'] = nowInSeconds() request = FibDataRequest(body=dataMap) self.assertTrue(request != None) self.assertTrue(request.fibId == dataMap['fib_id']) self.assertTrue(request.fibValue == dataMap['fib_value']) except: e = sys.exc_info()[0] print e self.fail(e)
def test5InitializeFromJSON(self): try: dataMap = {} dataMap['request_id'] ="foo1" dataMap['fib_id'] = 3 dataMap['fib_value'] = 3 dataMap['started_date'] = nowInSeconds() request = FibDataRequest(body = dataMap) self.assertTrue(request != None) self.assertTrue(request.fibId == dataMap['fib_id']) self.assertTrue(request.fibValue == dataMap['fib_value']) except: e = sys.exc_info()[0] print e self.fail(e)
def testGetPendingWorkItems(self): workerDataDB = initializeWorkerDataDB(self.testName) workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3}) workerDataDB.addWorkerData(workerData) workerData = WorkerData(None, {"request_id":"foo2","worker_id":"efgh","fib_id":5}) workerDataDB.addWorkerData(workerData) timestamp = nowInSeconds() workerDatas = workerDataDB.getWorkItems(isPending = True) for workerData in workerDatas: self.assertTrue(workerData.fibValue == -1) self.assertTrue(workerData.finishedDate == None) workerDatas = workerDataDB.getWorkItems(isPending = False) self.assertTrue(len(workerDatas) == 0)
def test2AddRequest(self): try: fibDataDB = initializeFibDataDB(self.testName) startedDate = nowInSeconds() fd = FibDataRequest(None, {"request_id":"foo1","fib_id":3,"fib_value":3,"started_date": startedDate}) newFd = fibDataDB.addRequest(fd) fetchedFd = fibDataDB.getRequest(newFd.requestId) self.assertTrue(fd.fibId == fetchedFd.fibId) self.assertTrue(fd.fibValue == fetchedFd.fibValue) self.assertTrue(fd.startedDate == fetchedFd.startedDate) except: e = sys.exc_info()[0] print e self.fail(e)
def __init__(self, row = None, body = None): if row != None: fibDataLogger.debug("initializing from database") self.requestId = row[0] self.fibId = row[1] self.fibValue = row[2] self.startedDate = row[3] self.finishedDate = row[4] elif body != None: fibDataLogger.debug("initializing from JSON") if body.has_key('request_id') == True: self.requestId = body['request_id'] else: self.requestId = -1 if body.has_key('fib_id') == True: self.fibId = int(body['fib_id']) else: fibDataLogger.error("invalid JSON format, fib_id not found") raise 'invalid format' if body.has_key('fib_value') == True: self.fibValue = int(body['fib_value']) else: self.fibValue = -1 if body.has_key('started_date'): self.startedDate = int(body['started_date']) else: self.startedDate = nowInSeconds() if body.has_key('finished_date'): self.finishedDate = int(body['finished_date']) else: self.finishedDate = None
def testGetCompletedWorkItems(self): workerDataDB = initializeWorkerDataDB(self.testName) startDate = nowInSeconds() - 5 workerData = WorkerData( None, { "request_id": "foo1", "worker_id": "abcd", "fib_id": 3, "startedDate": startDate }) workerDataDB.addWorkerData(workerData) workerData.fibValue = 3 workerDataDB.updateWorkerData(workerData) workerData = WorkerData( None, { "request_id": "foo2", "worker_id": "efgh", "fib_id": 5, "startedDate": startDate }) workerDataDB.addWorkerData(workerData) workerData.fibValue = 8 workerDataDB.updateWorkerData(workerData) workerDatas = workerDataDB.getWorkItems(isPending=True) self.assertTrue(len(workerDatas) == 0) workerDatas = workerDataDB.getWorkItems(isPending=False) for workerData in workerDatas: self.assertTrue(workerData.fibValue != -1) self.assertTrue(workerData.finishedDate != None) workerDatas = workerDataDB.getWorkItems(isPending=True) self.assertTrue(len(workerDatas) == 0)
def test13SerializeDisplayData(self): try: fibDataDB = initializeFibDataDB(self.testName) startedDate = nowInSeconds() fd = FibDataRequest( None, { "request_id": "foo1", "fib_id": 3, "fib_value": 3, "started_date": startedDate }) fd2 = FibDataRequest( None, { "request_id": "foo2", "fib_id": 4, "fib_value": 5, "started_date": startedDate }) fibDataDB.addRequest(fd) fibDataDB.addRequest(fd2) fdArr = fibDataDB.getRequests(isPending=True) self.assertTrue(len(fdArr) == 2) displayDataArr = [] for request in fdArr: displayDataArr.append(DisplayData(request)) str = json.dumps(displayDataArr, cls=DataEncoder) print str except: e = sys.exc_info()[0] print e self.fail(e)
def __init__(self, row=None, body=None): if row != None: fibDataLogger.debug("initializing from database") self.requestId = row[0] self.fibId = row[1] self.fibValue = row[2] self.startedDate = row[3] self.finishedDate = row[4] elif body != None: fibDataLogger.debug("initializing from JSON") if body.has_key('request_id') == True: self.requestId = body['request_id'] else: self.requestId = -1 if body.has_key('fib_id') == True: self.fibId = int(body['fib_id']) else: fibDataLogger.error("invalid JSON format, fib_id not found") raise 'invalid format' if body.has_key('fib_value') == True: self.fibValue = int(body['fib_value']) else: self.fibValue = -1 if body.has_key('started_date'): self.startedDate = int(body['started_date']) else: self.startedDate = nowInSeconds() if body.has_key('finished_date'): self.finishedDate = int(body['finished_date']) else: self.finishedDate = None
def addRequest(self, request): """ inserts a request into the database and timestamps it for readability """ try: db = self.connectToDB() cur = db.cursor() if request.startedDate == None: request.startedDate = nowInSeconds() if request.finishedDate == None: self.log.debug( "adding request into database with request_id = %s, fib_id = %d, fib_value = %d and started_date = %d" % (request.requestId, request.fibId, request.fibValue, request.startedDate)) query = "insert into fibdata(request_id,fib_id,fib_value,started_date) values('%s', %d, %d,%d)" % ( request.requestId, request.fibId, request.fibValue, request.startedDate) else: self.log.debug( "adding request into database with request_id = %s, fib_id = %d, fib_value = %d, started_date = %d, finished_date = %d" % (request.requestId, request.fibId, request.fibValue, request.startedDate, request.finishedDate)) query = "insert into fibdata(request_id,fib_id,fib_value,started_date,finished_date) values('%s',%d, %d,%d,%d)" % ( request.requestId, request.fibId, request.fibValue, request.startedDate, request.finishedDate) cur.execute(query) db.commit() self.disconnectFromDB(db) return request except MySQLdb.Error as e: self.log.error(str(e)) self.handleMySQLException(e)
def processMessage(message): ''' this is the handler function passed to getAndProcessMessages ''' dataMap = {} dataMap['worker_id'] = workerId dataMap['request_id'] = message.requestId dataMap['fib_id'] = message.messageKey dataMap['fib_value'] = -1 dataMap['started_date'] = message.startedDate workerData = WorkerData(body=dataMap) log.debug("worker %s starting Fibonnaci on %d at %s"%(workerId,workerData.fibId,prettyPrintTime(nowInSeconds())) ) addedWorkerData = workerDataDB.getWorkerData(requestId = workerData.requestId) if addedWorkerData != None: # if this is already in progress with another worker if addedWorkerData.workerId == workerId: log.error("expecting that this data was being worked on by a different worker, not %s"%workerId) workerData.retryCount = addedWorkerData.retryCount + 1 workerDataDB.updateWorkerData(workerData) else: addedWorkerData = workerDataDB.addWorkerData(workerData) log.debug("starting fib(%d)"%workerData.fibId) fibValue = F(message.messageKey) addedWorkerData.fibValue = fibValue log.debug("completed fib(%d), value = %d"%(workerData.fibId,fibValue)) addedWorkerData.finishedDate = nowInSeconds() workerDataDB.updateWorkerData(addedWorkerData) log.debug("worker %s finished Fibonnaci on %d, calculated value = %d, at %s"%(workerId,addedWorkerData.fibId,addedWorkerData.fibValue,prettyPrintTime(addedWorkerData.finishedDate)) ) # now send result back for result processing. resultsMessageQueue.sendMessage(WorkResultMessage(requestId = addedWorkerData.requestId, messageKey = addedWorkerData.fibId, messageValue = addedWorkerData.fibValue,startedDate = addedWorkerData.startedDate,finishedDate = addedWorkerData.finishedDate))
def updateRequest(self,request): """ updates a specified request, setting workerId, fibValue and finished_time values ONLY. (those are the things that can change) """ # explicit fail if the record hasn't been added if request.requestId == -1: raise 'cannot update a request that has not been added already' try: db = self.connectToDB() cur = db.cursor() finishedDate = nowInSeconds() self.log.debug("updating request with fib_id = %d and fib_value = %d and requestId = '%s'"%(request.fibId,request.fibValue,request.requestId)) query = "update fibdata set fib_value=%d,finished_date=%d where request_id = '%s'"%(request.fibValue,finishedDate, request.requestId) cur.execute(query) db.commit() self.disconnectFromDB(db) except MySQLdb.Error as e: self.log.error(str(e)) self.handleMySQLException(e)
def addWorkerData(self,workerData): """ inserts a workerData into the database and timestamps it for readability """ try: db = self.connectToDB() cur = db.cursor() self.log.debug("adding worker entry into database with request_id '%s', worker_id = '%s', fib_id=%d, startedDate = %s"%(workerData.requestId,workerData.workerId,workerData.fibId,nowInSeconds())) query = "insert into workerdata(request_id, worker_id, fib_id,started_date) values('%s','%s',%d,%d)"%(workerData.requestId,workerData.workerId,workerData.fibId,nowInSeconds()) cur.execute(query) db.commit() self.disconnectFromDB(db) return workerData except MySQLdb.Error as e: self.log.error(str(e)) self.handleMySQLException(e)
def test6AddRequestWithFinishedDate(self): try: fibDataDB = initializeFibDataDB(self.testName) fibData = FibDataRequest(None,{"request_id":"foo1","fib_id":3, "fib_value":2, "started_date" :nowInSeconds(), "finished_date" : nowInSeconds()}) newFD = fibDataDB.addRequest(fibData) self.assertTrue(newFD.requestId != -1) self.assertTrue(newFD.fibId == fibData.fibId) self.assertTrue(newFD.fibValue == fibData.fibValue) self.assertTrue(newFD.startedDate == fibData.startedDate) self.assertTrue(newFD.finishedDate == fibData.finishedDate) except: e = sys.exc_info()[0] print e self.fail(e)
def __init__(self, fibData): self.fibData = fibData self.formattedStartDate = prettyPrintTime(fibData.startedDate) if fibData.finishedDate != None: self.formattedFinishDate = prettyPrintTime(fibData.finishedDate) self.runTime = nowInSeconds() - fibData.startedDate
def updateWorkerData(self,workerData): """ updates a specified workerData, setting workerStatus and lastCheckinDate values ONLY. (those are the things that can change) """ try: db = self.connectToDB() cur = db.cursor() query = None if(workerData.fibValue > -1): self.log.debug("update workerdata set fib_value = %d, finished_date = %d, retry_count = %d where request_id='%s'"%(workerData.fibValue,nowInSeconds(), workerData.retryCount, workerData.requestId)) query = "update workerdata set fib_value = %d, finished_date = %d, retry_count = %d where request_id='%s'"%(workerData.fibValue,nowInSeconds(), workerData.retryCount,workerData.requestId) else: self.log.debug("update workerdata set fib_value = %d, started_date = %d, worker_id = '%s', retry_count = %d where request_id='%s'"%( workerData.fibValue,workerData.startedDate,workerData.workerId,workerData.retryCount, workerData.requestId)) query = "update workerdata set retry_count = %d where request_id='%s'"%(workerData.retryCount,workerData.requestId) cur.execute(query) db.commit() self.disconnectFromDB(db) except MySQLdb.Error as e: self.log.error(str(e)) self.handleMySQLException(e)