def sendFlare(self, messageData='awaken', flareDeadspaceSeconds=10): """ Sends a "flare" (a message to the pubsub topic path = self.topic_path) which will spark the SQL worker to jump into action. A flare will only be sent if there's been no flare sending in the last (flareDeadspaceSeconds) seconds (defaults to 10). """ flareSendable = False rightNow = datetime.now(timezone.utc) data = messageData.encode("utf-8") if self.redis_client: # Get the last flare-send time lastFlare = self.redis.get_project_human_val( 'SQLHandlerFlareSendTime') if not lastFlare: flareSendable = True # If the seconds-elapsed since this is longer than flareDeadspaceSeconds, send another one. else: eat it. elif mu.dateDiff('second', lastFlare, rightNow) >= flareDeadspaceSeconds: flareSendable = True if self.pubsub and flareSendable: self.redis.set_project_human_val('SQLHandlerFlareSendTime', 'datetime', rightNow) print(f'sending flare!') future = self.pubsub.publish(self.topic_path, data) future.result()
def goToWork(self, forHowLong=60, inactivityBuffer=10): print(f'XXX goToWork. ForHowLong={forHowLong}') startTs = datetime.now(timezone.utc) i = 0 howLong = 0 queuesAreEmpty = False while howLong <= forHowLong - inactivityBuffer and not queuesAreEmpty: i += 1 self.cypherQueues.getQLens() if self.cypherQueues.totalInWorkingQueue >= 10: self.lookForExpiredWorkingBlocks() ctb = self.popBlockFromWaitingQueues() if ctb: stm = ctb.statements print(f'XXX got CTB from Q statements = {stm}') print(ctb.transactionUid) self.executeBlock(ctb) else: queuesAreEmpty = True self.lookForExpiredWorkingBlocks() howLong = um.dateDiff('sec', startTs, datetime.now(timezone.utc)) print(f'Running for how long: {howLong}') if howLong >= forHowLong - inactivityBuffer and self.cypherQueues.totalInWaitingQueues > 0: # numFlares = self.cypherQueues.totalInWaitingQueues / 10 for k in range(3): print(f'sending flare (max 3) {k}') self.pubsub.publish_message('awaken') time.sleep(0.5)
def goToWork(self, forHowLong=60, inactivityBuffer=10, batchSize=50): print(f'XXX goingToWork. ForHowLong={forHowLong} s') priorities = ['H', 'M', 'L'] startTs = datetime.now() i = 0 howLong = 0 self.reportOnQueues() for priority in priorities: queuesAreEmpty = False while howLong <= forHowLong - inactivityBuffer and not queuesAreEmpty: i += 1 k = 0 batch = [] while not queuesAreEmpty and k < batchSize: sqlB, queuesAreEmpty = self.popNextBlock(priority=priority) if sqlB: batch.append(sqlB) k += 1 sortedBatches, transactionList = self.sortBatch(batch) for sb in sortedBatches: sqb = MonkeeSQLblock(query=sb[0], insertList=sb[1], numRetries=sb[2], maxRetries=sb[3], soloExecution=sb[4], lastExecAttempt=sb[5]) self.executeBlock(sqb, priority=priority) for transaction_i in transactionList: # transaction_i is a sqb, probably with more than one transactionSqb entry sqb = MonkeeSQLblock( query=transaction_i['query'], insertList=transaction_i['insertList'], numRetries=transaction_i['numRetries'], maxRetries=transaction_i['maxRetries'], soloExecution=transaction_i['soloExecution'], lastExecAttempt=transaction_i['lastExecAttempt'], transactionSqb=transaction_i['transactionSqb']) self.executeBlock(sqb, priority=priority) howLong = mu.dateDiff('sec', startTs, datetime.now()) #print(f'sqlw Running for how long: {howLong}') qlen = self.getQLens(priority=priority) if qlen == 0: queuesAreEmpty = True else: queuesAreEmpty = False if howLong >= forHowLong - inactivityBuffer and qlen > 0: numFlares = 3 for k in range(numFlares): self.sqlBHandler.sendFlare()
def _statementsAsTransaction(self, tx, statements): results = [] duration = [] statementList = [] for statement in statements: # try: startTs = datetime.now(timezone.utc) if "batch" in statement: cyph = statement["cypher"][0:100] print(f'batch-mode statement[cypher] to be run: {cyph}') res = tx.run(statement["cypher"], statement["parameters"], batch=statement["batch"]) else: cyph = statement["cypher"][0:100] print(f'statement[cypher] to be run: {cyph}') res = tx.run(statement["cypher"], statement["parameters"]) results.append(res) endTs = datetime.now(timezone.utc) duration.append(um.dateDiff('sec', startTs, endTs)) statementList.append({ "cypher": statement["cypher"], "parameters": statement["parameters"], "duration": um.dateDiff('sec', startTs, endTs), "status": "OK", "error": None }) # except Exception as e: # endTs = datetime.now(timezone.utc) # duration.append(um.dateDiff('sec', startTs, endTs)) # logging.error(repr(e)) # statementList.append({"cypher": statement["cypher"], "parameters": statement["parameters"], "duration": um.dateDiff('sec', startTs, endTs), # "status": "ERROR", "error": repr(e)}) return results, statementList
def lookForExpiredWorkingBlocks(self, expiryInSeconds=60): wb = self.copyBlockFromWorkingQueue() if wb: matchingSerial = json.dumps(wb.instanceToSerial(), cls=um.RoundTripEncoder) if wb.statements: timeDiff = um.dateDiff('s', wb.lastUpdatedAt, datetime.now()) print(f' Age of workingQ item ={timeDiff}') if timeDiff >= expiryInSeconds: stmt = wb.statements print( f'Picking up item from workingQ. Statements = {stmt}') rem = self.removeBlockFromWorkingQueue(matchingSerial) if rem > 0: wb.registerChangeInSql('outOfWorkingQ') wb.registerChangeInSql('RecycledToWaiting') wb.lastUpdatedAt = datetime.now() wb.numRetries += 1 self.cypherQueues.pushCtbToWaitingQ(ctBlock=wb, jumpTheQ=True)
def executeBlock(self, ctBlock: CypherTransactionBlock): """ Executes all statments in the ctb as one transaction. Returns success boolean """ matchingSerial = json.dumps(ctBlock.instanceToSerial(), cls=um.RoundTripEncoder) if ctBlock.statements: try: ctBlock.registerChangeInSql('executeStart') with self.neoDriver.session() as session: startTs = datetime.now(timezone.utc) print('to NEO: {}'.format(ctBlock.statements)) _, durations = session.write_transaction( self._statementsAsTransaction, ctBlock.statements) print('back from NEO') endTs = datetime.now(timezone.utc) elapsedSec = um.dateDiff('sec', startTs, endTs) ctBlock.runTime = elapsedSec ctBlock.durations = durations ctBlock.status = 'done' ctBlock.registerChangeInSql('executeEnd') rem = self.removeBlockFromWorkingQueue(matchingSerial) if rem > 0: ctBlock.registerChangeInSql('outOfWorkingQ') self.cypherQueues.pushCtbToCompletedQ(ctBlock) return True except CypherSyntaxError as e: print('CypherSyntaxError') logging.error(repr(e)) ctBlock.numRetries += 1 ctBlock.status = 'CypherSyntaxError' ctBlock.errors = repr(e) rem = self.removeBlockFromWorkingQueue(matchingSerial) if rem > 0: ctBlock.registerChangeInSql('outOfWorkingQ') self.cypherQueues.pushCtbToCompletedQ(ctBlock) ctBlock.registerChangeInSql('error', repr(e)) return False except ServiceUnavailable as e: print('ServiceUnavailable') logging.error(repr(e)) ctBlock.numRetries += 1 ctBlock.status = 'ServiceUnavailable' ctBlock.errors = repr(e) rem = self.removeBlockFromWorkingQueue(matchingSerial) if rem > 0: ctBlock.registerChangeInSql('outOfWorkingQ') self.cypherQueues.pushCtbToWaitingQ(ctBlock) ctBlock.registerChangeInSql('error', repr(e)) return False except ConstraintError as e: print('ConstraintError') logging.error(repr(e)) ctBlock.numRetries += 1 ctBlock.status = 'ConstraintError' ctBlock.errors = repr(e) rem = self.removeBlockFromWorkingQueue(matchingSerial) if rem > 0: ctBlock.registerChangeInSql('outOfWorkingQ') ctBlock.setJson() if ctBlock.numRetries <= 50: print(f'>>>>ctBlock.numRetries={ctBlock.numRetries}') self.cypherQueues.pushCtbToWaitingQ(ctBlock) ctBlock.registerChangeInSql('error', repr(e)) else: self.cypherQueues.pushCtbToCompletedQ(ctBlock) ctBlock.registerChangeInSql('givenUp', 'GIVEN UP') return False except ClientError as e: print('ClientError') logging.error(repr(e)) ctBlock.numRetries += 1 ctBlock.status = 'ClientError' ctBlock.errors = repr(e) rem = self.removeBlockFromWorkingQueue(matchingSerial) if rem > 0: ctBlock.registerChangeInSql('outOfWorkingQ') ctBlock.setJson() self.cypherQueues.pushCtbToCompletedQ(ctBlock) ctBlock.registerChangeInSql('error', repr(e)) return False except Exception as e: logging.error(repr(e)) rem = self.removeBlockFromWorkingQueue(matchingSerial) if rem > 0: ctBlock.registerChangeInSql('outOfWorkingQ') self.cypherQueues.pushCtbToWaitingQ(ctBlock) return False