def _runTask(
taskClass
, taskData
, taskConnection
, processorHome
, isThread = False
, useRConsole = False
):
def getLogForId(id):
return processorHome + '/logs/' + str(id) + '.log'
def getPidForId(id):
return processorHome + '/pids/' + str(id) + '.pid'
taskId = taskData['_id']
logFile = getLogForId(taskId)
pidFileName = getPidForId(taskId)
lastLogMessage = [ "(No log)" ]
def log(message):
now = datetime.datetime.utcnow().isoformat() + 'Z'
lastLogMessage[0] = message
showAs = now + ' ' + message
print(showAs)
with open(logFile, 'a') as f:
f.write(showAs + '\n')
# If anything goes wrong, default state is error
success = False
try:
# Set our python thread name
threading.current_thread().setName('lgTask-' + str(taskId))
# Launch rconsole?
if useRConsole:
import lgTask.lib.rfooUtil as rfooUtil
rfooUtil.spawnServer()
if not isThread:
# If we're not a thread, then we should change the process
# title to make ourselves identifiable, and also register
# a signal handler to convert SIGTERM into KillTaskError.
def onSigTerm(signum, frame):
raise KillTaskError()
signal.signal(signal.SIGTERM, onSigTerm)
conn = taskConnection
task = taskClass(conn, taskId, taskData)
if not isThread:
setProcessTitle(task, processorHome)
# Convert kwargs
kwargsOriginal = taskData['kwargs']
task._kwargsOriginal = kwargsOriginal
kwargs = conn._kwargsDecode(kwargsOriginal)
task.kwargs = kwargs
except:
log("Exception during init: " + traceback.format_exc())
else:
try:
task.log = log
# We tell the task where its log file is only so that fetchLogTask
# doesn't need to be smart, and can just find other tasks running
# on the same machine by simply changing the taskId in the
# filename.
task._lgTask_logFile = logFile
# Wait for the Processor to fill out our pid file; this ensures
# that if the processor crashes after setting our state to working
# and after launching us, then it can justifiably claim that we
# have died when it does not find the pid file.
waitFor = 5 #seconds
sleepTime = 0.1
while True:
with open(pidFileName, 'r') as f:
if f.read() == str(os.getpid()):
break
waitFor -= sleepTime
if waitFor <= 0:
raise Exception("Pid file never initialized")
time.sleep(sleepTime)
if task.DEBUG_TIMING:
log("Starting task at {0}".format(
datetime.datetime.utcnow().isoformat()
))
try:
success = task.run(**kwargs)
finally:
task._Task_finalize()
if success is None:
# No return value means success
success = True
except RetryTaskError, e:
log("Retrying task after {0}".format(e.delay))
success = e
except:
def run(self):
# If we can, replace lgTaskProcessor with lgTaskSlave in our title
try:
import setproctitle
title = setproctitle.getproctitle()
if 'lgTaskProcessor' in title:
title = title.replace('lgTaskProcessor', 'lgTaskSlave')
else:
title += ' --slave'
setproctitle.setproctitle(title)
except ImportError:
pass
# We're in our own process now, so disconnect the processor's
# pymongo connection to make sure we don't hold those sockets open
self._processor.taskConnection.close()
# Also, ensure that the global talk variables weren't copied over.
# This only affects testing situations - that is, the normal processor
# process won't use talk.
import lgTask.talk
lgTask.talk.talkConnection.resetFork()
canQsize = True
try:
self._queue.qsize()
except NotImplementedError:
# Oh Mac OS X, how silly you are sometimes
canQsize = False
self._fixSigTerm()
# rconsole?
if self._processor._useRConsole:
import lgTask.lib.rfooUtil as rfooUtil
rfooUtil.spawnServer()
# Any tasks that we start only really need a teeny bit of stack
thread.stack_size(1024 * 1024)
try:
while True:
try:
# See if we should be marked as accepting new tasks from
# the Processor
if self._isAccepting.value:
self._checkAccepting()
# Check tasks are running
self._checkRunning()
# Get new task
taskData = self._queue.get(
timeout = self._processor.KILL_INTERVAL
)
taskThread = InterruptableThread(
target = self._runTaskThreadMain
, args = (taskData,)
)
# Remember the ID so that we can check for "kill" states
taskThread.taskId = taskData['_id']
taskThread.start()
self._running.append(taskThread)
# Update running count
newCount = len(self._running)
if canQsize:
newCount += self._queue.qsize()
self._runningCount.value = newCount
except Empty:
pass
except Exception:
self._processor.log("Slave error {0}: {1}".format(
self.pid, traceback.format_exc()
))
# After each iteration, see if we're alive
if not self._shouldContinue():
break
except:
self._processor.log("Slave error {0}: {1}".format(
self.pid, traceback.format_exc()
))
finally:
pass
