def _runTask( taskClass , taskData , taskConnection , processorHome , isThread = False , useRConsole = False ): def getLogForId(id): return processorHome + '/logs/' + str(id) + '.log' def getPidForId(id): return processorHome + '/pids/' + str(id) + '.pid' taskId = taskData['_id'] logFile = getLogForId(taskId) pidFileName = getPidForId(taskId) lastLogMessage = [ "(No log)" ] def log(message): now = datetime.datetime.utcnow().isoformat() + 'Z' lastLogMessage[0] = message showAs = now + ' ' + message print(showAs) with open(logFile, 'a') as f: f.write(showAs + '\n') # If anything goes wrong, default state is error success = False try: # Set our python thread name threading.current_thread().setName('lgTask-' + str(taskId)) # Launch rconsole? if useRConsole: import lgTask.lib.rfooUtil as rfooUtil rfooUtil.spawnServer() if not isThread: # If we're not a thread, then we should change the process # title to make ourselves identifiable, and also register # a signal handler to convert SIGTERM into KillTaskError. def onSigTerm(signum, frame): raise KillTaskError() signal.signal(signal.SIGTERM, onSigTerm) conn = taskConnection task = taskClass(conn, taskId, taskData) if not isThread: setProcessTitle(task, processorHome) # Convert kwargs kwargsOriginal = taskData['kwargs'] task._kwargsOriginal = kwargsOriginal kwargs = conn._kwargsDecode(kwargsOriginal) task.kwargs = kwargs except: log("Exception during init: " + traceback.format_exc()) else: try: task.log = log # We tell the task where its log file is only so that fetchLogTask # doesn't need to be smart, and can just find other tasks running # on the same machine by simply changing the taskId in the # filename. task._lgTask_logFile = logFile # Wait for the Processor to fill out our pid file; this ensures # that if the processor crashes after setting our state to working # and after launching us, then it can justifiably claim that we # have died when it does not find the pid file. waitFor = 5 #seconds sleepTime = 0.1 while True: with open(pidFileName, 'r') as f: if f.read() == str(os.getpid()): break waitFor -= sleepTime if waitFor <= 0: raise Exception("Pid file never initialized") time.sleep(sleepTime) if task.DEBUG_TIMING: log("Starting task at {0}".format( datetime.datetime.utcnow().isoformat() )) try: success = task.run(**kwargs) finally: task._Task_finalize() if success is None: # No return value means success success = True except RetryTaskError, e: log("Retrying task after {0}".format(e.delay)) success = e except:
def run(self): # If we can, replace lgTaskProcessor with lgTaskSlave in our title try: import setproctitle title = setproctitle.getproctitle() if 'lgTaskProcessor' in title: title = title.replace('lgTaskProcessor', 'lgTaskSlave') else: title += ' --slave' setproctitle.setproctitle(title) except ImportError: pass # We're in our own process now, so disconnect the processor's # pymongo connection to make sure we don't hold those sockets open self._processor.taskConnection.close() # Also, ensure that the global talk variables weren't copied over. # This only affects testing situations - that is, the normal processor # process won't use talk. import lgTask.talk lgTask.talk.talkConnection.resetFork() canQsize = True try: self._queue.qsize() except NotImplementedError: # Oh Mac OS X, how silly you are sometimes canQsize = False self._fixSigTerm() # rconsole? if self._processor._useRConsole: import lgTask.lib.rfooUtil as rfooUtil rfooUtil.spawnServer() # Any tasks that we start only really need a teeny bit of stack thread.stack_size(1024 * 1024) try: while True: try: # See if we should be marked as accepting new tasks from # the Processor if self._isAccepting.value: self._checkAccepting() # Check tasks are running self._checkRunning() # Get new task taskData = self._queue.get( timeout = self._processor.KILL_INTERVAL ) taskThread = InterruptableThread( target = self._runTaskThreadMain , args = (taskData,) ) # Remember the ID so that we can check for "kill" states taskThread.taskId = taskData['_id'] taskThread.start() self._running.append(taskThread) # Update running count newCount = len(self._running) if canQsize: newCount += self._queue.qsize() self._runningCount.value = newCount except Empty: pass except Exception: self._processor.log("Slave error {0}: {1}".format( self.pid, traceback.format_exc() )) # After each iteration, see if we're alive if not self._shouldContinue(): break except: self._processor.log("Slave error {0}: {1}".format( self.pid, traceback.format_exc() )) finally: pass