def _periodicTryEmptyQueryBacklog(self):
"""Will periodically check if the query backlog has any entries in it, and if so, try to issue those
pending queries against the DB.
"""
if singleton.get('castdaemon', strict=False) and (singleton.get('castdaemon').isShuttingDown() or errorutil.checkIfFatalErrorOccurred()):
return
if not self.getQueryBacklogSize():
return
#print "PERIODIC trying to empty backlog, size is", self.getQueryBacklogSize()
d = self._emptyBacklog()
return d
def _directProcessExecute_onError(failure, queryString, argList, fetch, connID, useCache,
cacheExpireTime, cacheHashKey, printQuery, _alreadyInBacklog):
if failure.check(MySQLdb.ProgrammingError) or failure.check(TypeError):
log.msg(u"Database query failure. Error: %s. Failed query was: %s; Args: (%s)"
% (failure.getErrorMessage(), queryString, ', '.join([x for x in argList]),), lvl='e', ss='ss_db')
failure.raiseException() #invalid syntax error
elif failure.check(MySQLdb.OperationalError):
if singleton.get('castdaemon', strict=False) and (singleton.get('castdaemon').isShuttingDown() or errorutil.checkIfFatalErrorOccurred()):
#we shouldn't try to queue queries when the server is going down (as this may hold up the shutdown
# process)
failure.raiseException()
if _isMySQLServerConnDownErrorMessage(failure.getErrorMessage()) and not _alreadyInBacklog:
#currently not connected, queue up the request to be run when the connection is restored
dRequestCompleted = defer.Deferred() #will be fired when the query is finally issued against the DB
backlogItem = {'callType': 'execute', 'queryString': queryString, 'argList': argList,
'fetch': fetch, 'connID': connID, 'useCache': useCache,
'cacheExpireTime': cacheExpireTime, 'cacheHashKey': cacheHashKey, 'printQuery': printQuery,
'dRequestCompleted': dRequestCompleted, }
log.msg(u"Connectivity failure: Queuing query (MD5: %s): \"%s\". New backlog length: %i"
% (_getBacklogEntryLoggingHash(backlogItem),
sharedstrutil.truncate(sharedstrutil.removeMultipleSpacesFromString(queryString).strip(), 80),
singleton.get(connID).getQueryBacklogSize() + 1), lvl='i', ss='ss_db')
singleton.get(connID)._addToQueryBacklog(backlogItem)
return backlogItem['dRequestCompleted']
elif _isMySQLServerConnDownErrorMessage(failure.getErrorMessage()) and _alreadyInBacklog:
#don't insert the entry in to the backlog again as it's already there
#print "dbExecute: CONNECTION STILL DOWN!"
return EXECUTE_ALREADY_QUEUED_RETURNDATA
else:
log.msg(u"Unknown database operational error. Error: %s. Failed query was: %s; Args: (%s)"
% (failure.getErrorMessage(), queryString,
', '.join([x for x in argList]),), lvl='e', ss='ss_db')
failure.raiseException()
def _periodicConnPing(self):
"""
Pings across all connections on the database. Used to keep the connectivity alive and prevent idle
timeouts on the MySQL server.
"""
if singleton.get('castdaemon', strict=False) and (singleton.get('castdaemon').isShuttingDown() or errorutil.checkIfFatalErrorOccurred()):
return
cannotPingOnConns = []
firstExceptionValue = None
for conn in self.connections.values():
try:
conn.ping()
except MySQLdb.OperationalError:
excType, excValue, excTraceback = sys.exc_info()
#try to close this connection
try:
self._close(conn)
except MySQLdb.ProgrammingError: #closing a closed connection
#connection is already closed, don't include it in our list of bad connections we had
pass
cannotPingOnConns.append(conn)
if not firstExceptionValue:
firstExceptionValue = excValue
if len(cannotPingOnConns):
log.msg(u"Could not ping MySQL server for pool \"%s\" on %i of %i pool connections: %s. Reset these connections..." % (
self._poolName, len(cannotPingOnConns), len(self.connections.values()), firstExceptionValue),
lvl='w', ss='ss_db')
