def sendFlare(self, messageData='awaken', flareDeadspaceSeconds=10):
"""
Sends a "flare" (a message to the pubsub topic path = self.topic_path) which will spark the SQL worker to jump into action.
A flare will only be sent if there's been no flare sending in the last (flareDeadspaceSeconds) seconds (defaults to 10).
"""
flareSendable = False
rightNow = datetime.now(timezone.utc)
data = messageData.encode("utf-8")
if self.redis_client:
# Get the last flare-send time
lastFlare = self.redis.get_project_human_val(
'SQLHandlerFlareSendTime')
if not lastFlare:
flareSendable = True
# If the seconds-elapsed since this is longer than flareDeadspaceSeconds, send another one. else: eat it.
elif mu.dateDiff('second', lastFlare,
rightNow) >= flareDeadspaceSeconds:
flareSendable = True
if self.pubsub and flareSendable:
self.redis.set_project_human_val('SQLHandlerFlareSendTime',
'datetime', rightNow)
print(f'sending flare!')
future = self.pubsub.publish(self.topic_path, data)
future.result()
def goToWork(self, forHowLong=60, inactivityBuffer=10):
print(f'XXX goToWork. ForHowLong={forHowLong}')
startTs = datetime.now(timezone.utc)
i = 0
howLong = 0
queuesAreEmpty = False
while howLong <= forHowLong - inactivityBuffer and not queuesAreEmpty:
i += 1
self.cypherQueues.getQLens()
if self.cypherQueues.totalInWorkingQueue >= 10:
self.lookForExpiredWorkingBlocks()
ctb = self.popBlockFromWaitingQueues()
if ctb:
stm = ctb.statements
print(f'XXX got CTB from Q statements = {stm}')
print(ctb.transactionUid)
self.executeBlock(ctb)
else:
queuesAreEmpty = True
self.lookForExpiredWorkingBlocks()
howLong = um.dateDiff('sec', startTs, datetime.now(timezone.utc))
print(f'Running for how long: {howLong}')
if howLong >= forHowLong - inactivityBuffer and self.cypherQueues.totalInWaitingQueues > 0:
# numFlares = self.cypherQueues.totalInWaitingQueues / 10
for k in range(3):
print(f'sending flare (max 3) {k}')
self.pubsub.publish_message('awaken')
time.sleep(0.5)
def goToWork(self, forHowLong=60, inactivityBuffer=10, batchSize=50):
print(f'XXX goingToWork. ForHowLong={forHowLong} s')
priorities = ['H', 'M', 'L']
startTs = datetime.now()
i = 0
howLong = 0
self.reportOnQueues()
for priority in priorities:
queuesAreEmpty = False
while howLong <= forHowLong - inactivityBuffer and not queuesAreEmpty:
i += 1
k = 0
batch = []
while not queuesAreEmpty and k < batchSize:
sqlB, queuesAreEmpty = self.popNextBlock(priority=priority)
if sqlB:
batch.append(sqlB)
k += 1
sortedBatches, transactionList = self.sortBatch(batch)
for sb in sortedBatches:
sqb = MonkeeSQLblock(query=sb[0],
insertList=sb[1],
numRetries=sb[2],
maxRetries=sb[3],
soloExecution=sb[4],
lastExecAttempt=sb[5])
self.executeBlock(sqb, priority=priority)
for transaction_i in transactionList:
# transaction_i is a sqb, probably with more than one transactionSqb entry
sqb = MonkeeSQLblock(
query=transaction_i['query'],
insertList=transaction_i['insertList'],
numRetries=transaction_i['numRetries'],
maxRetries=transaction_i['maxRetries'],
soloExecution=transaction_i['soloExecution'],
lastExecAttempt=transaction_i['lastExecAttempt'],
transactionSqb=transaction_i['transactionSqb'])
self.executeBlock(sqb, priority=priority)
howLong = mu.dateDiff('sec', startTs, datetime.now())
#print(f'sqlw Running for how long: {howLong}')
qlen = self.getQLens(priority=priority)
if qlen == 0:
queuesAreEmpty = True
else:
queuesAreEmpty = False
if howLong >= forHowLong - inactivityBuffer and qlen > 0:
numFlares = 3
for k in range(numFlares):
self.sqlBHandler.sendFlare()
def _statementsAsTransaction(self, tx, statements):
results = []
duration = []
statementList = []
for statement in statements:
# try:
startTs = datetime.now(timezone.utc)
if "batch" in statement:
cyph = statement["cypher"][0:100]
print(f'batch-mode statement[cypher] to be run: {cyph}')
res = tx.run(statement["cypher"],
statement["parameters"],
batch=statement["batch"])
else:
cyph = statement["cypher"][0:100]
print(f'statement[cypher] to be run: {cyph}')
res = tx.run(statement["cypher"], statement["parameters"])
results.append(res)
endTs = datetime.now(timezone.utc)
duration.append(um.dateDiff('sec', startTs, endTs))
statementList.append({
"cypher": statement["cypher"],
"parameters": statement["parameters"],
"duration": um.dateDiff('sec', startTs, endTs),
"status": "OK",
"error": None
})
# except Exception as e:
# endTs = datetime.now(timezone.utc)
# duration.append(um.dateDiff('sec', startTs, endTs))
# logging.error(repr(e))
# statementList.append({"cypher": statement["cypher"], "parameters": statement["parameters"], "duration": um.dateDiff('sec', startTs, endTs),
# "status": "ERROR", "error": repr(e)})
return results, statementList
def lookForExpiredWorkingBlocks(self, expiryInSeconds=60):
wb = self.copyBlockFromWorkingQueue()
if wb:
matchingSerial = json.dumps(wb.instanceToSerial(),
cls=um.RoundTripEncoder)
if wb.statements:
timeDiff = um.dateDiff('s', wb.lastUpdatedAt, datetime.now())
print(f' Age of workingQ item ={timeDiff}')
if timeDiff >= expiryInSeconds:
stmt = wb.statements
print(
f'Picking up item from workingQ. Statements = {stmt}')
rem = self.removeBlockFromWorkingQueue(matchingSerial)
if rem > 0:
wb.registerChangeInSql('outOfWorkingQ')
wb.registerChangeInSql('RecycledToWaiting')
wb.lastUpdatedAt = datetime.now()
wb.numRetries += 1
self.cypherQueues.pushCtbToWaitingQ(ctBlock=wb,
jumpTheQ=True)
def executeBlock(self, ctBlock: CypherTransactionBlock):
"""
Executes all statments in the ctb as one transaction.
Returns success boolean
"""
matchingSerial = json.dumps(ctBlock.instanceToSerial(),
cls=um.RoundTripEncoder)
if ctBlock.statements:
try:
ctBlock.registerChangeInSql('executeStart')
with self.neoDriver.session() as session:
startTs = datetime.now(timezone.utc)
print('to NEO: {}'.format(ctBlock.statements))
_, durations = session.write_transaction(
self._statementsAsTransaction, ctBlock.statements)
print('back from NEO')
endTs = datetime.now(timezone.utc)
elapsedSec = um.dateDiff('sec', startTs, endTs)
ctBlock.runTime = elapsedSec
ctBlock.durations = durations
ctBlock.status = 'done'
ctBlock.registerChangeInSql('executeEnd')
rem = self.removeBlockFromWorkingQueue(matchingSerial)
if rem > 0:
ctBlock.registerChangeInSql('outOfWorkingQ')
self.cypherQueues.pushCtbToCompletedQ(ctBlock)
return True
except CypherSyntaxError as e:
print('CypherSyntaxError')
logging.error(repr(e))
ctBlock.numRetries += 1
ctBlock.status = 'CypherSyntaxError'
ctBlock.errors = repr(e)
rem = self.removeBlockFromWorkingQueue(matchingSerial)
if rem > 0:
ctBlock.registerChangeInSql('outOfWorkingQ')
self.cypherQueues.pushCtbToCompletedQ(ctBlock)
ctBlock.registerChangeInSql('error', repr(e))
return False
except ServiceUnavailable as e:
print('ServiceUnavailable')
logging.error(repr(e))
ctBlock.numRetries += 1
ctBlock.status = 'ServiceUnavailable'
ctBlock.errors = repr(e)
rem = self.removeBlockFromWorkingQueue(matchingSerial)
if rem > 0:
ctBlock.registerChangeInSql('outOfWorkingQ')
self.cypherQueues.pushCtbToWaitingQ(ctBlock)
ctBlock.registerChangeInSql('error', repr(e))
return False
except ConstraintError as e:
print('ConstraintError')
logging.error(repr(e))
ctBlock.numRetries += 1
ctBlock.status = 'ConstraintError'
ctBlock.errors = repr(e)
rem = self.removeBlockFromWorkingQueue(matchingSerial)
if rem > 0:
ctBlock.registerChangeInSql('outOfWorkingQ')
ctBlock.setJson()
if ctBlock.numRetries <= 50:
print(f'>>>>ctBlock.numRetries={ctBlock.numRetries}')
self.cypherQueues.pushCtbToWaitingQ(ctBlock)
ctBlock.registerChangeInSql('error', repr(e))
else:
self.cypherQueues.pushCtbToCompletedQ(ctBlock)
ctBlock.registerChangeInSql('givenUp', 'GIVEN UP')
return False
except ClientError as e:
print('ClientError')
logging.error(repr(e))
ctBlock.numRetries += 1
ctBlock.status = 'ClientError'
ctBlock.errors = repr(e)
rem = self.removeBlockFromWorkingQueue(matchingSerial)
if rem > 0:
ctBlock.registerChangeInSql('outOfWorkingQ')
ctBlock.setJson()
self.cypherQueues.pushCtbToCompletedQ(ctBlock)
ctBlock.registerChangeInSql('error', repr(e))
return False
except Exception as e:
logging.error(repr(e))
rem = self.removeBlockFromWorkingQueue(matchingSerial)
if rem > 0:
ctBlock.registerChangeInSql('outOfWorkingQ')
self.cypherQueues.pushCtbToWaitingQ(ctBlock)
return False