def sortBatch(self, batch): batchDict = {} batchList = [] transactionList = [] for line in batch: isTransaction = mu.getval(line, "isTransaction", 0) # The reason that non-transactions are broken up out of their sqbs is so that similar queries can be batched, # meaning that all sqbs in this batch where the query part is the same, but only the parameters differ are bundled together and executed as one. if isTransaction == 0: query = mu.getval(line, "query") soloExecution = mu.getval(line, "soloExecution", 0) numRetries = mu.getval(line, "numRetries", 0) maxRetries = mu.getval(line, "maxRetries", 0) lastExecAttempt = mu.getval(line, "lastExecAttempt") if query: if soloExecution == 0: if query not in batchDict: batchDict[query] = [] batchDict[query].append(line["insertList"]) elif soloExecution == 1: # soloExecution = flagging this element to be executed on its own, i.e. not as part of a batch batchList.append([ query, line["insertList"], numRetries, maxRetries, soloExecution, lastExecAttempt ]) elif isTransaction == 1: # if this is a transaction, there is no batching. This implies that the sqb (containing the transaction) can be passed through directly. transactionList.append(line) for q in batchDict: batchList.append([q, batchDict[q], 0, 30, 0, lastExecAttempt]) return batchList, transactionList
def popNextBlock(self, priority): if priority == 'H': theQ = self.sqlBHandler.sqlQname_H elif priority == 'M': theQ = self.sqlBHandler.sqlQname_M elif priority == 'L': theQ = self.sqlBHandler.sqlQname_L elif priority == 'ERR': theQ = self.sqlBHandler.sqlQname_ERR popped = self.sqlBHandler.redis_client.blpop(theQ, 1) if not popped: print( f"SQL_Q {priority} is EMPTY_________________________________________" ) else: dataFromRedis = json.loads(popped[1], cls=mu.RoundTripDecoder) numRetries = mu.getval(dataFromRedis, "numRetries", 0) lastExecAttempt = mu.getval(dataFromRedis, "lastExecAttempt") remainInErrorQUntil = mu.getval(dataFromRedis, "remainInErrorQUntil", datetime.now()) if numRetries == 0: return dataFromRedis, False # elif lastExecAttempt and datetime.now() >= lastExecAttempt + timedelta(seconds=1.5 ** numRetries): elif priority in [ 'H', 'M', 'L' ] and lastExecAttempt and datetime.now( ) >= lastExecAttempt + timedelta(seconds=2 * numRetries): return dataFromRedis, False elif priority in ['ERR'] and remainInErrorQUntil and datetime.now( ) >= remainInErrorQUntil: return dataFromRedis, False else: sqlB = MonkeeSQLblock() sqlB.makeFromSerial(serial_=dataFromRedis) self.sqlBHandler.toQ(sqlB, priority=priority) return None, True
def makeFromSerial(self, serial_): self.isTransaction = mu.getval(serial_, "isTransaction", 0) if self.isTransaction == 0: self.query = mu.getval(serial_, "query") self.insertList = mu.getval(serial_, "insertList") self.queryTypeId = mu.getval(serial_, "queryTypeId") self.numRetries = mu.getval(serial_, "numRetries") self.maxRetries = mu.getval(serial_, "maxRetries") self.soloExecution = mu.getval(serial_, "soloExecution") self.lastExecAttempt = mu.getval(serial_, "lastExecAttempt") self.serial_ = self.instanceToSerial() elif self.isTransaction == 1: self.statements = self.query = mu.getval(serial_, "statements", []) for statement in self.statements: sqb = MonkeeSQLblock() sqb.query = mu.getval(statement, "query") sqb.insertList = mu.getval(statement, "insertList") sqb.queryTypeId = mu.getval(statement, "queryTypeId") sqb.numRetries = mu.getval(statement, "numRetries") sqb.maxRetries = mu.getval(statement, "maxRetries") sqb.soloExecution = mu.getval(statement, "soloExecution") sqb.lastExecAttempt = mu.getval(statement, "lastExecAttempt") sqb.serial_ = sqb.instanceToSerial() self.transactionSqb.append(sqb)
def makeFromSerial(self, serial_): self.isTransaction = mu.getval(serial_, "isTransaction", 0) if self.isTransaction == 0: self.query = mu.getval(serial_, "query") self.insertList = mu.getval(serial_, "insertList") self.queryTypeId = mu.getval(serial_, "queryTypeId") self.numRetries = mu.getval(serial_, "numRetries") self.maxRetries = mu.getval(serial_, "maxRetries") self.soloExecution = mu.getval(serial_, "soloExecution") self.lastExecAttempt = mu.getval(serial_, "lastExecAttempt") self.priority = mu.getval(serial_, "priority") self.remainInErrorQUntil = mu.getval(serial_, "remainInErrorQUntil") self.serial_ = self.instanceToSerial() elif self.isTransaction == 1: self.statements = mu.getval(serial_, "statements", []) self.transactionSqb = mu.getval(serial_, "transactionSqb", []) self.numRetries = mu.getval(serial_, "numRetries") self.maxRetries = mu.getval(serial_, "maxRetries") self.lastExecAttempt = mu.getval(serial_, "lastExecAttempt") self.priority = mu.getval(serial_, "priority") self.remainInErrorQUntil = mu.getval(serial_, "remainInErrorQUntil") if len(self.statements) > 0 and len(self.transactionSqb) == 0: for statement in self.statements: sqb = MonkeeSQLblock() sqb.query = mu.getval(statement, "query") sqb.insertList = mu.getval(statement, "insertList") sqb.queryTypeId = mu.getval(statement, "queryTypeId") sqb.numRetries = mu.getval(statement, "numRetries") sqb.maxRetries = mu.getval(statement, "maxRetries") sqb.soloExecution = mu.getval(statement, "soloExecution") sqb.lastExecAttempt = mu.getval(statement, "lastExecAttempt") sqb.serial_ = sqb.instanceToSerial() self.transactionSqb.append(sqb) self.instanceToSerial()
def makeFromSerial(self, serial): if not isinstance(serial, dict): dict_ = json.loads(serial, cls=um.RoundTripDecoder) else: dict_ = serial self.priority = um.getval(dict_, "priority") self.numRetries = um.getval(dict_, "numRetries") self.createdAt = um.getval(dict_, "createdAt") self.lastUpdatedAt = um.getval(dict_, "lastUpdatedAt") self.transactionUid = um.getval(dict_, "uid") self.origin = um.getval(dict_, "origin") self.statements = um.getval(dict_, "statements") self.runTime = um.getval(dict_, "runTime") self.status = um.getval(dict_, "status") self.errors = um.getval(dict_, "errors") self.durations = um.getval(dict_, "durations") self.callingCF = um.getval(dict_, "callingCF") self.timeInQ = um.getval(dict_, "timeInQ") self.docUid = um.getval(dict_, "docUid") self.appUid = um.getval(dict_, "appUid") self.setJson()