def __init__(self, requestId = None, messageKey = None, body = None):
messageLogger.debug("initializing from JSON")
if body != None:
if body.has_key('requestId') == True:
self.requestId = body['requestId']
else:
messageLogger.error("invalid JSON format, requestId not found")
raise 'invalid format'
if body.has_key('messageKey') == True:
self.messageKey = body['messageKey']
else:
messageLogger.error("invalid JSON format, messageKey not found")
raise 'invalid format'
# created is optional.
if body.has_key('startedDate'):
self.starteDate = body['startedDate']
else:
self.startedDate = nowInSeconds()
else:
self.requestId = requestId
self.messageKey = messageKey
self.createdDate = nowInSeconds()
def updateWorkerData(self, workerData):
"""
updates a specified workerData, setting workerStatus and lastCheckinDate values ONLY.
(those are the things that can change)
"""
try:
db = self.connectToDB()
cur = db.cursor()
query = None
if (workerData.fibValue > -1):
self.log.debug(
"update workerdata set fib_value = %d, finished_date = %d, retry_count = %d where request_id='%s'"
% (workerData.fibValue, nowInSeconds(),
workerData.retryCount, workerData.requestId))
query = "update workerdata set fib_value = %d, finished_date = %d, retry_count = %d where request_id='%s'" % (
workerData.fibValue, nowInSeconds(), workerData.retryCount,
workerData.requestId)
else:
self.log.debug(
"update workerdata set fib_value = %d, started_date = %d, worker_id = '%s', retry_count = %d where request_id='%s'"
% (workerData.fibValue, workerData.startedDate,
workerData.workerId, workerData.retryCount,
workerData.requestId))
query = "update workerdata set retry_count = %d where request_id='%s'" % (
workerData.retryCount, workerData.requestId)
cur.execute(query)
db.commit()
self.disconnectFromDB(db)
except MySQLdb.Error as e:
self.log.error(str(e))
self.handleMySQLException(e)
def addWorkerData(self, workerData):
"""
inserts a workerData into the database and timestamps it for readability
"""
try:
db = self.connectToDB()
cur = db.cursor()
self.log.debug(
"adding worker entry into database with request_id '%s', worker_id = '%s', fib_id=%d, startedDate = %s"
% (workerData.requestId, workerData.workerId, workerData.fibId,
nowInSeconds()))
query = "insert into workerdata(request_id, worker_id, fib_id,started_date) values('%s','%s',%d,%d)" % (
workerData.requestId, workerData.workerId, workerData.fibId,
nowInSeconds())
cur.execute(query)
db.commit()
self.disconnectFromDB(db)
return workerData
except MySQLdb.Error as e:
self.log.error(str(e))
self.handleMySQLException(e)
def __init__(self, requestId=None, messageKey=None, body=None):
messageLogger.debug("initializing from JSON")
if body != None:
if body.has_key('requestId') == True:
self.requestId = body['requestId']
else:
messageLogger.error("invalid JSON format, requestId not found")
raise 'invalid format'
if body.has_key('messageKey') == True:
self.messageKey = body['messageKey']
else:
messageLogger.error(
"invalid JSON format, messageKey not found")
raise 'invalid format'
# created is optional.
if body.has_key('startedDate'):
self.starteDate = body['startedDate']
else:
self.startedDate = nowInSeconds()
else:
self.requestId = requestId
self.messageKey = messageKey
self.createdDate = nowInSeconds()
def test12Add_UpdateRequest_FetchAllCompleteTasks(self):
fibDataDB = initializeFibDataDB(self.testName)
finishedDate = nowInSeconds()
startedDate = finishedDate - 5
fd = FibDataRequest(
None, {
"request_id": "foo1",
"fib_id": 3,
"fib_value": 3,
"started_date": startedDate,
"finished_date": finishedDate
})
fibDataDB.addRequest(fd)
fdArr = fibDataDB.getRequests(isPending=False)
self.assertTrue(len(fdArr) == 1)
startedDate = nowInSeconds()
fd = FibDataRequest(
None, {
"request_id": "foo2",
"fib_id": 3,
"fib_value": 3,
"started_date": startedDate
})
fibDataDB.addRequest(fd)
fdArr = fibDataDB.getRequests(isPending=False)
self.assertTrue(len(fdArr) == 1)
def processMessage(message):
'''
this is the handler function passed to getAndProcessMessages
'''
dataMap = {}
dataMap['worker_id'] = workerId
dataMap['request_id'] = message.requestId
dataMap['fib_id'] = message.messageKey
dataMap['fib_value'] = -1
dataMap['started_date'] = message.startedDate
workerData = WorkerData(body=dataMap)
log.debug(
"worker %s starting Fibonnaci on %d at %s" %
(workerId, workerData.fibId, prettyPrintTime(nowInSeconds())))
addedWorkerData = workerDataDB.getWorkerData(
requestId=workerData.requestId)
if addedWorkerData != None:
# if this is already in progress with another worker
if addedWorkerData.workerId == workerId:
log.error(
"expecting that this data was being worked on by a different worker, not %s"
% workerId)
workerData.retryCount = addedWorkerData.retryCount + 1
workerDataDB.updateWorkerData(workerData)
else:
addedWorkerData = workerDataDB.addWorkerData(workerData)
log.debug("starting fib(%d)" % workerData.fibId)
fibValue = F(message.messageKey)
addedWorkerData.fibValue = fibValue
log.debug("completed fib(%d), value = %d" %
(workerData.fibId, fibValue))
addedWorkerData.finishedDate = nowInSeconds()
workerDataDB.updateWorkerData(addedWorkerData)
log.debug(
"worker %s finished Fibonnaci on %d, calculated value = %d, at %s"
% (workerId, addedWorkerData.fibId, addedWorkerData.fibValue,
prettyPrintTime(addedWorkerData.finishedDate)))
# now send result back for result processing.
resultsMessageQueue.sendMessage(
WorkResultMessage(requestId=addedWorkerData.requestId,
messageKey=addedWorkerData.fibId,
messageValue=addedWorkerData.fibValue,
startedDate=addedWorkerData.startedDate,
finishedDate=addedWorkerData.finishedDate))
def testGetPendingWorkItems(self):
workerDataDB = initializeWorkerDataDB(self.testName)
workerData = WorkerData(None, {
"request_id": "foo1",
"worker_id": "abcd",
"fib_id": 3
})
workerDataDB.addWorkerData(workerData)
workerData = WorkerData(None, {
"request_id": "foo2",
"worker_id": "efgh",
"fib_id": 5
})
workerDataDB.addWorkerData(workerData)
timestamp = nowInSeconds()
workerDatas = workerDataDB.getWorkItems(isPending=True)
for workerData in workerDatas:
self.assertTrue(workerData.fibValue == -1)
self.assertTrue(workerData.finishedDate == None)
workerDatas = workerDataDB.getWorkItems(isPending=False)
self.assertTrue(len(workerDatas) == 0)
def __init__(self, workerData):
self.workerData = workerData
self.formattedStartDate = prettyPrintTime(workerData.startedDate)
if workerData.finishedDate != None:
self.formattedFinishDate = prettyPrintTime(workerData.finishedDate)
else:
self.runTime = nowInSeconds() - workerData.startedDate
def test3GetAllRequests(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
startedDate = nowInSeconds()
fd = FibDataRequest(
None, {
"request_id": "foo2",
"fib_id": 3,
"fib_value": 3,
"started_date": startedDate
})
newFd = fibDataDB.addRequest(fd)
fdArr = fibDataDB.getRequests(isPending=True)
self.assertTrue(len(fdArr) > 0)
self.assertTrue(len(fdArr) == 1)
testFd = fdArr[0]
self.assertTrue(testFd.fibId == newFd.fibId)
fdArr2 = fibDataDB.getRequests(isPending=True, isDescending=False)
self.assertTrue(len(fdArr2) == 1)
testFd2 = fdArr2[0]
self.assertTrue(testFd2.fibId == newFd.fibId)
fdArr3 = fibDataDB.getRequests()
self.assertTrue(len(fdArr3) == 0)
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def __init__(self, workerData):
self.workerData = workerData
self.formattedStartDate = prettyPrintTime(workerData.startedDate)
if workerData.finishedDate != None:
self.formattedFinishDate = prettyPrintTime(workerData.finishedDate)
else:
self.runTime = nowInSeconds() - workerData.startedDate
def test13SerializeDisplayData(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
startedDate = nowInSeconds()
fd = FibDataRequest(None, {"request_id":"foo1","fib_id":3,"fib_value":3,"started_date": startedDate})
fd2 = FibDataRequest(None,{"request_id":"foo2","fib_id":4,"fib_value":5,"started_date": startedDate})
fibDataDB.addRequest(fd)
fibDataDB.addRequest(fd2)
fdArr = fibDataDB.getRequests(isPending = True)
self.assertTrue(len(fdArr) == 2)
displayDataArr = []
for request in fdArr:
displayDataArr.append(DisplayData(request))
str = json.dumps(displayDataArr,cls=DataEncoder)
print str
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def addRequest(self,request):
"""
inserts a request into the database and timestamps it for readability
"""
try:
db = self.connectToDB()
cur = db.cursor()
if request.startedDate == None:
request.startedDate = nowInSeconds()
if request.finishedDate == None:
self.log.debug("adding request into database with request_id = %s, fib_id = %d, fib_value = %d and started_date = %d"%(request.requestId,request.fibId,request.fibValue,request.startedDate))
query = "insert into fibdata(request_id,fib_id,fib_value,started_date) values('%s', %d, %d,%d)"%( request.requestId,request.fibId, request.fibValue,request.startedDate)
else:
self.log.debug("adding request into database with request_id = %s, fib_id = %d, fib_value = %d, started_date = %d, finished_date = %d"%(request.requestId,request.fibId,request.fibValue,request.startedDate,request.finishedDate))
query = "insert into fibdata(request_id,fib_id,fib_value,started_date,finished_date) values('%s',%d, %d,%d,%d)"%(request.requestId,request.fibId, request.fibValue,request.startedDate,request.finishedDate)
cur.execute(query)
db.commit()
self.disconnectFromDB(db)
return request
except MySQLdb.Error as e:
self.log.error(str(e))
self.handleMySQLException(e)
def updateRequest(self, request):
"""
updates a specified request, setting workerId, fibValue and finished_time values ONLY.
(those are the things that can change)
"""
# explicit fail if the record hasn't been added
if request.requestId == -1:
raise 'cannot update a request that has not been added already'
try:
db = self.connectToDB()
cur = db.cursor()
finishedDate = nowInSeconds()
self.log.debug(
"updating request with fib_id = %d and fib_value = %d and requestId = '%s'"
% (request.fibId, request.fibValue, request.requestId))
query = "update fibdata set fib_value=%d,finished_date=%d where request_id = '%s'" % (
request.fibValue, finishedDate, request.requestId)
cur.execute(query)
db.commit()
self.disconnectFromDB(db)
except MySQLdb.Error as e:
self.log.error(str(e))
self.handleMySQLException(e)
def testGetCompletedWorkItems(self):
workerDataDB = initializeWorkerDataDB(self.testName)
startDate = nowInSeconds()- 5;
workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3,"startedDate":startDate})
workerDataDB.addWorkerData(workerData)
workerData.fibValue = 3
workerDataDB.updateWorkerData(workerData)
workerData = WorkerData(None, {"request_id":"foo2","worker_id":"efgh","fib_id":5,"startedDate":startDate})
workerDataDB.addWorkerData(workerData)
workerData.fibValue = 8
workerDataDB.updateWorkerData(workerData)
workerDatas = workerDataDB.getWorkItems(isPending = True)
self.assertTrue(len(workerDatas) == 0)
workerDatas = workerDataDB.getWorkItems(isPending = False)
for workerData in workerDatas:
self.assertTrue(workerData.fibValue != -1)
self.assertTrue(workerData.finishedDate != None)
workerDatas = workerDataDB.getWorkItems(isPending = True)
self.assertTrue(len(workerDatas) == 0)
def test2AddRequest(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
startedDate = nowInSeconds()
fd = FibDataRequest(
None, {
"request_id": "foo1",
"fib_id": 3,
"fib_value": 3,
"started_date": startedDate
})
newFd = fibDataDB.addRequest(fd)
fetchedFd = fibDataDB.getRequest(newFd.requestId)
self.assertTrue(fd.fibId == fetchedFd.fibId)
self.assertTrue(fd.fibValue == fetchedFd.fibValue)
self.assertTrue(fd.startedDate == fetchedFd.startedDate)
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def test3GetAllRequests(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
startedDate = nowInSeconds()
fd = FibDataRequest(None, {"request_id":"foo2","fib_id":3,"fib_value":3,"started_date": startedDate})
newFd = fibDataDB.addRequest(fd)
fdArr = fibDataDB.getRequests(isPending = True)
self.assertTrue(len(fdArr) > 0)
self.assertTrue(len(fdArr) == 1)
testFd = fdArr[0]
self.assertTrue(testFd.fibId == newFd.fibId)
fdArr2 = fibDataDB.getRequests(isPending = True,isDescending=False)
self.assertTrue(len(fdArr2) == 1)
testFd2 = fdArr2[0]
self.assertTrue(testFd2.fibId == newFd.fibId)
fdArr3 = fibDataDB.getRequests()
self.assertTrue(len(fdArr3) == 0)
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def __init__(self, row = None, body = None):
# SELECT id,request_id, worker_id,fib_id,fib_value, started_date,finished_date
if row != None:
workerDataLogger.debug("initializing from database")
self.requestId = row[0]
self.workerId = row[1]
self.fibId = row[2]
self.fibValue = row[3]
self.retryCount = row[4]
self.startedDate = row[5]
if len(row) == 7:
self.finishedDate = row[6]
else:
self.finishedDate = None
elif body != None:
workerDataLogger.debug("initializing from JSON")
if body.has_key('request_id') == True:
self.requestId = body['request_id']
else:
workerDataLogger.error("invalid JSON format, request_id not found")
raise 'invalid format'
if body.has_key('worker_id') == True:
self.workerId = body['worker_id']
else:
workerDataLogger.error("invalid JSON format, worker_id not found")
raise 'invalid format'
if body.has_key('fib_id') == True:
self.fibId = body['fib_id']
else:
workerDataLogger.error("invalid JSON format, fib_id not found")
raise 'invalid format'
if body.has_key('fib_value'):
self.fibValue = body['fib_value']
else:
self.fibValue = -1
if body.has_key("retry_count"):
self.finishedDate = body['retry_count']
else:
self.retryCount = 0
if body.has_key("started_date"):
self.startedDate = body['started_date']
else:
self.startedDate = nowInSeconds()
if body.has_key("finished_date"):
self.finishedDate = body['finished_date']
else:
self.finishedDate = None
def __init__(self, row=None, body=None):
# SELECT id,request_id, worker_id,fib_id,fib_value, started_date,finished_date
if row != None:
workerDataLogger.debug("initializing from database")
self.requestId = row[0]
self.workerId = row[1]
self.fibId = row[2]
self.fibValue = row[3]
self.retryCount = row[4]
self.startedDate = row[5]
if len(row) == 7:
self.finishedDate = row[6]
else:
self.finishedDate = None
elif body != None:
workerDataLogger.debug("initializing from JSON")
if body.has_key('request_id') == True:
self.requestId = body['request_id']
else:
workerDataLogger.error(
"invalid JSON format, request_id not found")
raise 'invalid format'
if body.has_key('worker_id') == True:
self.workerId = body['worker_id']
else:
workerDataLogger.error(
"invalid JSON format, worker_id not found")
raise 'invalid format'
if body.has_key('fib_id') == True:
self.fibId = body['fib_id']
else:
workerDataLogger.error("invalid JSON format, fib_id not found")
raise 'invalid format'
if body.has_key('fib_value'):
self.fibValue = body['fib_value']
else:
self.fibValue = -1
if body.has_key("retry_count"):
self.finishedDate = body['retry_count']
else:
self.retryCount = 0
if body.has_key("started_date"):
self.startedDate = body['started_date']
else:
self.startedDate = nowInSeconds()
if body.has_key("finished_date"):
self.finishedDate = body['finished_date']
else:
self.finishedDate = None
def test12Add_UpdateRequest_FetchAllCompleteTasks(self):
fibDataDB = initializeFibDataDB(self.testName)
finishedDate = nowInSeconds()
startedDate = finishedDate - 5;
fd = FibDataRequest(None, {"request_id":"foo1","fib_id":3,"fib_value":3,"started_date": startedDate,"finished_date":finishedDate})
fibDataDB.addRequest(fd)
fdArr = fibDataDB.getRequests(isPending = False)
self.assertTrue(len(fdArr) == 1)
startedDate = nowInSeconds()
fd = FibDataRequest(None, {"request_id":"foo2","fib_id":3,"fib_value":3,"started_date": startedDate})
fibDataDB.addRequest(fd)
fdArr = fibDataDB.getRequests(isPending = False)
self.assertTrue(len(fdArr) == 1)
def testUpdateWorkerData(self):
workerDataDB = initializeWorkerDataDB(self.testName)
lastCheckinDate = nowInSeconds()
lastCheckinDate = lastCheckinDate - 5
workerData = WorkerData(
None, {
"request_id": "foo1",
"worker_id": "abcd",
"fib_id": 3,
"started_date": nowInSeconds() - 5
})
newWorkerData = workerDataDB.addWorkerData(workerData)
workerData.fibValue = 3
workerDataDB.updateWorkerData(workerData)
checkWorkerData = workerDataDB.getWorkerData(workerData.workerId)
self.assertTrue(checkWorkerData.finishedDate != None)
def test6AddRequestWithFinishedDate(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
fibData = FibDataRequest(
None, {
"request_id": "foo1",
"fib_id": 3,
"fib_value": 2,
"started_date": nowInSeconds(),
"finished_date": nowInSeconds()
})
newFD = fibDataDB.addRequest(fibData)
self.assertTrue(newFD.requestId != -1)
self.assertTrue(newFD.fibId == fibData.fibId)
self.assertTrue(newFD.fibValue == fibData.fibValue)
self.assertTrue(newFD.startedDate == fibData.startedDate)
self.assertTrue(newFD.finishedDate == fibData.finishedDate)
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def test7Add_UpdateRequest(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
fibData = FibDataRequest(
None, {
"request_id": "foo1",
"fib_id": 3,
"fib_value": 2,
"started_date": nowInSeconds(),
"finished_date": nowInSeconds()
})
newFD = fibDataDB.addRequest(fibData)
finishedDate = nowInSeconds() + 5
fibData.finishedDate = finishedDate
fibDataDB.updateRequest(fibData)
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def testUpdateWorkerData(self):
workerDataDB = initializeWorkerDataDB(self.testName)
lastCheckinDate = nowInSeconds()
lastCheckinDate = lastCheckinDate-5
workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3,"started_date":nowInSeconds()-5})
newWorkerData = workerDataDB.addWorkerData(workerData)
workerData.fibValue=3
workerDataDB.updateWorkerData(workerData)
checkWorkerData = workerDataDB.getWorkerData(workerData.workerId)
self.assertTrue(checkWorkerData.finishedDate != None)
def test7Add_UpdateRequest(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
fibData = FibDataRequest(None,{"request_id":"foo1","fib_id":3, "fib_value":2, "started_date" :nowInSeconds(), "finished_date" : nowInSeconds()})
newFD = fibDataDB.addRequest(fibData)
finishedDate = nowInSeconds()+5
fibData.finishedDate = finishedDate
fibDataDB.updateRequest(fibData)
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def test5InitializeFromJSON(self):
try:
dataMap = {}
dataMap['request_id'] = "foo1"
dataMap['fib_id'] = 3
dataMap['fib_value'] = 3
dataMap['started_date'] = nowInSeconds()
request = FibDataRequest(body=dataMap)
self.assertTrue(request != None)
self.assertTrue(request.fibId == dataMap['fib_id'])
self.assertTrue(request.fibValue == dataMap['fib_value'])
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def test5InitializeFromJSON(self):
try:
dataMap = {}
dataMap['request_id'] ="foo1"
dataMap['fib_id'] = 3
dataMap['fib_value'] = 3
dataMap['started_date'] = nowInSeconds()
request = FibDataRequest(body = dataMap)
self.assertTrue(request != None)
self.assertTrue(request.fibId == dataMap['fib_id'])
self.assertTrue(request.fibValue == dataMap['fib_value'])
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def testGetPendingWorkItems(self):
workerDataDB = initializeWorkerDataDB(self.testName)
workerData = WorkerData(None, {"request_id":"foo1","worker_id":"abcd","fib_id":3})
workerDataDB.addWorkerData(workerData)
workerData = WorkerData(None, {"request_id":"foo2","worker_id":"efgh","fib_id":5})
workerDataDB.addWorkerData(workerData)
timestamp = nowInSeconds()
workerDatas = workerDataDB.getWorkItems(isPending = True)
for workerData in workerDatas:
self.assertTrue(workerData.fibValue == -1)
self.assertTrue(workerData.finishedDate == None)
workerDatas = workerDataDB.getWorkItems(isPending = False)
self.assertTrue(len(workerDatas) == 0)
def test2AddRequest(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
startedDate = nowInSeconds()
fd = FibDataRequest(None, {"request_id":"foo1","fib_id":3,"fib_value":3,"started_date": startedDate})
newFd = fibDataDB.addRequest(fd)
fetchedFd = fibDataDB.getRequest(newFd.requestId)
self.assertTrue(fd.fibId == fetchedFd.fibId)
self.assertTrue(fd.fibValue == fetchedFd.fibValue)
self.assertTrue(fd.startedDate == fetchedFd.startedDate)
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def __init__(self, row = None, body = None):
if row != None:
fibDataLogger.debug("initializing from database")
self.requestId = row[0]
self.fibId = row[1]
self.fibValue = row[2]
self.startedDate = row[3]
self.finishedDate = row[4]
elif body != None:
fibDataLogger.debug("initializing from JSON")
if body.has_key('request_id') == True:
self.requestId = body['request_id']
else:
self.requestId = -1
if body.has_key('fib_id') == True:
self.fibId = int(body['fib_id'])
else:
fibDataLogger.error("invalid JSON format, fib_id not found")
raise 'invalid format'
if body.has_key('fib_value') == True:
self.fibValue = int(body['fib_value'])
else:
self.fibValue = -1
if body.has_key('started_date'):
self.startedDate = int(body['started_date'])
else:
self.startedDate = nowInSeconds()
if body.has_key('finished_date'):
self.finishedDate = int(body['finished_date'])
else:
self.finishedDate = None
def testGetCompletedWorkItems(self):
workerDataDB = initializeWorkerDataDB(self.testName)
startDate = nowInSeconds() - 5
workerData = WorkerData(
None, {
"request_id": "foo1",
"worker_id": "abcd",
"fib_id": 3,
"startedDate": startDate
})
workerDataDB.addWorkerData(workerData)
workerData.fibValue = 3
workerDataDB.updateWorkerData(workerData)
workerData = WorkerData(
None, {
"request_id": "foo2",
"worker_id": "efgh",
"fib_id": 5,
"startedDate": startDate
})
workerDataDB.addWorkerData(workerData)
workerData.fibValue = 8
workerDataDB.updateWorkerData(workerData)
workerDatas = workerDataDB.getWorkItems(isPending=True)
self.assertTrue(len(workerDatas) == 0)
workerDatas = workerDataDB.getWorkItems(isPending=False)
for workerData in workerDatas:
self.assertTrue(workerData.fibValue != -1)
self.assertTrue(workerData.finishedDate != None)
workerDatas = workerDataDB.getWorkItems(isPending=True)
self.assertTrue(len(workerDatas) == 0)
def test13SerializeDisplayData(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
startedDate = nowInSeconds()
fd = FibDataRequest(
None, {
"request_id": "foo1",
"fib_id": 3,
"fib_value": 3,
"started_date": startedDate
})
fd2 = FibDataRequest(
None, {
"request_id": "foo2",
"fib_id": 4,
"fib_value": 5,
"started_date": startedDate
})
fibDataDB.addRequest(fd)
fibDataDB.addRequest(fd2)
fdArr = fibDataDB.getRequests(isPending=True)
self.assertTrue(len(fdArr) == 2)
displayDataArr = []
for request in fdArr:
displayDataArr.append(DisplayData(request))
str = json.dumps(displayDataArr, cls=DataEncoder)
print str
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def __init__(self, row=None, body=None):
if row != None:
fibDataLogger.debug("initializing from database")
self.requestId = row[0]
self.fibId = row[1]
self.fibValue = row[2]
self.startedDate = row[3]
self.finishedDate = row[4]
elif body != None:
fibDataLogger.debug("initializing from JSON")
if body.has_key('request_id') == True:
self.requestId = body['request_id']
else:
self.requestId = -1
if body.has_key('fib_id') == True:
self.fibId = int(body['fib_id'])
else:
fibDataLogger.error("invalid JSON format, fib_id not found")
raise 'invalid format'
if body.has_key('fib_value') == True:
self.fibValue = int(body['fib_value'])
else:
self.fibValue = -1
if body.has_key('started_date'):
self.startedDate = int(body['started_date'])
else:
self.startedDate = nowInSeconds()
if body.has_key('finished_date'):
self.finishedDate = int(body['finished_date'])
else:
self.finishedDate = None
def addRequest(self, request):
"""
inserts a request into the database and timestamps it for readability
"""
try:
db = self.connectToDB()
cur = db.cursor()
if request.startedDate == None:
request.startedDate = nowInSeconds()
if request.finishedDate == None:
self.log.debug(
"adding request into database with request_id = %s, fib_id = %d, fib_value = %d and started_date = %d"
% (request.requestId, request.fibId, request.fibValue,
request.startedDate))
query = "insert into fibdata(request_id,fib_id,fib_value,started_date) values('%s', %d, %d,%d)" % (
request.requestId, request.fibId, request.fibValue,
request.startedDate)
else:
self.log.debug(
"adding request into database with request_id = %s, fib_id = %d, fib_value = %d, started_date = %d, finished_date = %d"
% (request.requestId, request.fibId, request.fibValue,
request.startedDate, request.finishedDate))
query = "insert into fibdata(request_id,fib_id,fib_value,started_date,finished_date) values('%s',%d, %d,%d,%d)" % (
request.requestId, request.fibId, request.fibValue,
request.startedDate, request.finishedDate)
cur.execute(query)
db.commit()
self.disconnectFromDB(db)
return request
except MySQLdb.Error as e:
self.log.error(str(e))
self.handleMySQLException(e)
def processMessage(message):
'''
this is the handler function passed to getAndProcessMessages
'''
dataMap = {}
dataMap['worker_id'] = workerId
dataMap['request_id'] = message.requestId
dataMap['fib_id'] = message.messageKey
dataMap['fib_value'] = -1
dataMap['started_date'] = message.startedDate
workerData = WorkerData(body=dataMap)
log.debug("worker %s starting Fibonnaci on %d at %s"%(workerId,workerData.fibId,prettyPrintTime(nowInSeconds())) )
addedWorkerData = workerDataDB.getWorkerData(requestId = workerData.requestId)
if addedWorkerData != None:
# if this is already in progress with another worker
if addedWorkerData.workerId == workerId:
log.error("expecting that this data was being worked on by a different worker, not %s"%workerId)
workerData.retryCount = addedWorkerData.retryCount + 1
workerDataDB.updateWorkerData(workerData)
else:
addedWorkerData = workerDataDB.addWorkerData(workerData)
log.debug("starting fib(%d)"%workerData.fibId)
fibValue = F(message.messageKey)
addedWorkerData.fibValue = fibValue
log.debug("completed fib(%d), value = %d"%(workerData.fibId,fibValue))
addedWorkerData.finishedDate = nowInSeconds()
workerDataDB.updateWorkerData(addedWorkerData)
log.debug("worker %s finished Fibonnaci on %d, calculated value = %d, at %s"%(workerId,addedWorkerData.fibId,addedWorkerData.fibValue,prettyPrintTime(addedWorkerData.finishedDate)) )
# now send result back for result processing.
resultsMessageQueue.sendMessage(WorkResultMessage(requestId = addedWorkerData.requestId, messageKey = addedWorkerData.fibId, messageValue = addedWorkerData.fibValue,startedDate = addedWorkerData.startedDate,finishedDate = addedWorkerData.finishedDate))
def updateRequest(self,request):
"""
updates a specified request, setting workerId, fibValue and finished_time values ONLY.
(those are the things that can change)
"""
# explicit fail if the record hasn't been added
if request.requestId == -1:
raise 'cannot update a request that has not been added already'
try:
db = self.connectToDB()
cur = db.cursor()
finishedDate = nowInSeconds()
self.log.debug("updating request with fib_id = %d and fib_value = %d and requestId = '%s'"%(request.fibId,request.fibValue,request.requestId))
query = "update fibdata set fib_value=%d,finished_date=%d where request_id = '%s'"%(request.fibValue,finishedDate, request.requestId)
cur.execute(query)
db.commit()
self.disconnectFromDB(db)
except MySQLdb.Error as e:
self.log.error(str(e))
self.handleMySQLException(e)
def addWorkerData(self,workerData):
"""
inserts a workerData into the database and timestamps it for readability
"""
try:
db = self.connectToDB()
cur = db.cursor()
self.log.debug("adding worker entry into database with request_id '%s', worker_id = '%s', fib_id=%d, startedDate = %s"%(workerData.requestId,workerData.workerId,workerData.fibId,nowInSeconds()))
query = "insert into workerdata(request_id, worker_id, fib_id,started_date) values('%s','%s',%d,%d)"%(workerData.requestId,workerData.workerId,workerData.fibId,nowInSeconds())
cur.execute(query)
db.commit()
self.disconnectFromDB(db)
return workerData
except MySQLdb.Error as e:
self.log.error(str(e))
self.handleMySQLException(e)
def test6AddRequestWithFinishedDate(self):
try:
fibDataDB = initializeFibDataDB(self.testName)
fibData = FibDataRequest(None,{"request_id":"foo1","fib_id":3, "fib_value":2, "started_date" :nowInSeconds(), "finished_date" : nowInSeconds()})
newFD = fibDataDB.addRequest(fibData)
self.assertTrue(newFD.requestId != -1)
self.assertTrue(newFD.fibId == fibData.fibId)
self.assertTrue(newFD.fibValue == fibData.fibValue)
self.assertTrue(newFD.startedDate == fibData.startedDate)
self.assertTrue(newFD.finishedDate == fibData.finishedDate)
except:
e = sys.exc_info()[0]
print e
self.fail(e)
def __init__(self, fibData):
self.fibData = fibData
self.formattedStartDate = prettyPrintTime(fibData.startedDate)
if fibData.finishedDate != None:
self.formattedFinishDate = prettyPrintTime(fibData.finishedDate)
self.runTime = nowInSeconds() - fibData.startedDate
def updateWorkerData(self,workerData):
"""
updates a specified workerData, setting workerStatus and lastCheckinDate values ONLY.
(those are the things that can change)
"""
try:
db = self.connectToDB()
cur = db.cursor()
query = None
if(workerData.fibValue > -1):
self.log.debug("update workerdata set fib_value = %d, finished_date = %d, retry_count = %d where request_id='%s'"%(workerData.fibValue,nowInSeconds(), workerData.retryCount, workerData.requestId))
query = "update workerdata set fib_value = %d, finished_date = %d, retry_count = %d where request_id='%s'"%(workerData.fibValue,nowInSeconds(), workerData.retryCount,workerData.requestId)
else:
self.log.debug("update workerdata set fib_value = %d, started_date = %d, worker_id = '%s', retry_count = %d where request_id='%s'"%( workerData.fibValue,workerData.startedDate,workerData.workerId,workerData.retryCount, workerData.requestId))
query = "update workerdata set retry_count = %d where request_id='%s'"%(workerData.retryCount,workerData.requestId)
cur.execute(query)
db.commit()
self.disconnectFromDB(db)
except MySQLdb.Error as e:
self.log.error(str(e))
self.handleMySQLException(e)
def __init__(self, fibData):
self.fibData = fibData
self.formattedStartDate = prettyPrintTime(fibData.startedDate)
if fibData.finishedDate != None:
self.formattedFinishDate = prettyPrintTime(fibData.finishedDate)
self.runTime = nowInSeconds() - fibData.startedDate
