def sortBatch(self, batch):
batchDict = {}
batchList = []
transactionList = []
for line in batch:
isTransaction = mu.getval(line, "isTransaction", 0)
# The reason that non-transactions are broken up out of their sqbs is so that similar queries can be batched,
# meaning that all sqbs in this batch where the query part is the same, but only the parameters differ are bundled together and executed as one.
if isTransaction == 0:
query = mu.getval(line, "query")
soloExecution = mu.getval(line, "soloExecution", 0)
numRetries = mu.getval(line, "numRetries", 0)
maxRetries = mu.getval(line, "maxRetries", 0)
lastExecAttempt = mu.getval(line, "lastExecAttempt")
if query:
if soloExecution == 0:
if query not in batchDict:
batchDict[query] = []
batchDict[query].append(line["insertList"])
elif soloExecution == 1: # soloExecution = flagging this element to be executed on its own, i.e. not as part of a batch
batchList.append([
query, line["insertList"], numRetries, maxRetries,
soloExecution, lastExecAttempt
])
elif isTransaction == 1:
# if this is a transaction, there is no batching. This implies that the sqb (containing the transaction) can be passed through directly.
transactionList.append(line)
for q in batchDict:
batchList.append([q, batchDict[q], 0, 30, 0, lastExecAttempt])
return batchList, transactionList
def popNextBlock(self, priority):
if priority == 'H':
theQ = self.sqlBHandler.sqlQname_H
elif priority == 'M':
theQ = self.sqlBHandler.sqlQname_M
elif priority == 'L':
theQ = self.sqlBHandler.sqlQname_L
elif priority == 'ERR':
theQ = self.sqlBHandler.sqlQname_ERR
popped = self.sqlBHandler.redis_client.blpop(theQ, 1)
if not popped:
print(
f"SQL_Q {priority} is EMPTY_________________________________________"
)
else:
dataFromRedis = json.loads(popped[1], cls=mu.RoundTripDecoder)
numRetries = mu.getval(dataFromRedis, "numRetries", 0)
lastExecAttempt = mu.getval(dataFromRedis, "lastExecAttempt")
remainInErrorQUntil = mu.getval(dataFromRedis,
"remainInErrorQUntil",
datetime.now())
if numRetries == 0:
return dataFromRedis, False
# elif lastExecAttempt and datetime.now() >= lastExecAttempt + timedelta(seconds=1.5 ** numRetries):
elif priority in [
'H', 'M', 'L'
] and lastExecAttempt and datetime.now(
) >= lastExecAttempt + timedelta(seconds=2 * numRetries):
return dataFromRedis, False
elif priority in ['ERR'] and remainInErrorQUntil and datetime.now(
) >= remainInErrorQUntil:
return dataFromRedis, False
else:
sqlB = MonkeeSQLblock()
sqlB.makeFromSerial(serial_=dataFromRedis)
self.sqlBHandler.toQ(sqlB, priority=priority)
return None, True
def makeFromSerial(self, serial_):
self.isTransaction = mu.getval(serial_, "isTransaction", 0)
if self.isTransaction == 0:
self.query = mu.getval(serial_, "query")
self.insertList = mu.getval(serial_, "insertList")
self.queryTypeId = mu.getval(serial_, "queryTypeId")
self.numRetries = mu.getval(serial_, "numRetries")
self.maxRetries = mu.getval(serial_, "maxRetries")
self.soloExecution = mu.getval(serial_, "soloExecution")
self.lastExecAttempt = mu.getval(serial_, "lastExecAttempt")
self.serial_ = self.instanceToSerial()
elif self.isTransaction == 1:
self.statements = self.query = mu.getval(serial_, "statements", [])
for statement in self.statements:
sqb = MonkeeSQLblock()
sqb.query = mu.getval(statement, "query")
sqb.insertList = mu.getval(statement, "insertList")
sqb.queryTypeId = mu.getval(statement, "queryTypeId")
sqb.numRetries = mu.getval(statement, "numRetries")
sqb.maxRetries = mu.getval(statement, "maxRetries")
sqb.soloExecution = mu.getval(statement, "soloExecution")
sqb.lastExecAttempt = mu.getval(statement, "lastExecAttempt")
sqb.serial_ = sqb.instanceToSerial()
self.transactionSqb.append(sqb)
def makeFromSerial(self, serial_):
self.isTransaction = mu.getval(serial_, "isTransaction", 0)
if self.isTransaction == 0:
self.query = mu.getval(serial_, "query")
self.insertList = mu.getval(serial_, "insertList")
self.queryTypeId = mu.getval(serial_, "queryTypeId")
self.numRetries = mu.getval(serial_, "numRetries")
self.maxRetries = mu.getval(serial_, "maxRetries")
self.soloExecution = mu.getval(serial_, "soloExecution")
self.lastExecAttempt = mu.getval(serial_, "lastExecAttempt")
self.priority = mu.getval(serial_, "priority")
self.remainInErrorQUntil = mu.getval(serial_,
"remainInErrorQUntil")
self.serial_ = self.instanceToSerial()
elif self.isTransaction == 1:
self.statements = mu.getval(serial_, "statements", [])
self.transactionSqb = mu.getval(serial_, "transactionSqb", [])
self.numRetries = mu.getval(serial_, "numRetries")
self.maxRetries = mu.getval(serial_, "maxRetries")
self.lastExecAttempt = mu.getval(serial_, "lastExecAttempt")
self.priority = mu.getval(serial_, "priority")
self.remainInErrorQUntil = mu.getval(serial_,
"remainInErrorQUntil")
if len(self.statements) > 0 and len(self.transactionSqb) == 0:
for statement in self.statements:
sqb = MonkeeSQLblock()
sqb.query = mu.getval(statement, "query")
sqb.insertList = mu.getval(statement, "insertList")
sqb.queryTypeId = mu.getval(statement, "queryTypeId")
sqb.numRetries = mu.getval(statement, "numRetries")
sqb.maxRetries = mu.getval(statement, "maxRetries")
sqb.soloExecution = mu.getval(statement, "soloExecution")
sqb.lastExecAttempt = mu.getval(statement,
"lastExecAttempt")
sqb.serial_ = sqb.instanceToSerial()
self.transactionSqb.append(sqb)
self.instanceToSerial()
def makeFromSerial(self, serial):
if not isinstance(serial, dict):
dict_ = json.loads(serial, cls=um.RoundTripDecoder)
else:
dict_ = serial
self.priority = um.getval(dict_, "priority")
self.numRetries = um.getval(dict_, "numRetries")
self.createdAt = um.getval(dict_, "createdAt")
self.lastUpdatedAt = um.getval(dict_, "lastUpdatedAt")
self.transactionUid = um.getval(dict_, "uid")
self.origin = um.getval(dict_, "origin")
self.statements = um.getval(dict_, "statements")
self.runTime = um.getval(dict_, "runTime")
self.status = um.getval(dict_, "status")
self.errors = um.getval(dict_, "errors")
self.durations = um.getval(dict_, "durations")
self.callingCF = um.getval(dict_, "callingCF")
self.timeInQ = um.getval(dict_, "timeInQ")
self.docUid = um.getval(dict_, "docUid")
self.appUid = um.getval(dict_, "appUid")
self.setJson()