def launchRenderTask(self, HydraJob, HydraTask): """Does the actual rendering, then records the results on the database""" logger.info("Starting task with id %s on job with id %s", HydraTask.id, HydraJob.id) self.HydraJob = HydraJob self.HydraTask = HydraTask self.childKilled = 0 self.statusAfterDeath = None self.childProcess = None self.PSUtilProc = None originalCurrentFrame = int(self.HydraTask.currentFrame) renderTaskCMD = self.HydraTask.createTaskCMD(self.HydraJob, sys.platform) logger.debug(renderTaskCMD) self.logPath = self.HydraTask.getLogPath() logger.info("Starting render task %s", self.HydraTask.id) try: log = file(self.logPath, 'w') except (IOError, OSError, WindowsError) as e: logger.error(e) self.thisNode.getOff() return log.write('Hydra log file {0} on {1}\n'.format(self.logPath, self.HydraTask.host)) log.write('RenderNode is {0}\n'.format(sys.argv)) log.write('Command: {0}\n\n'.format(renderTaskCMD)) Utils.flushOut(log) progressUpdateThread = stoppableThread(self.progressUpdate, 300, "Progress_Update_Thread") #Run the job and keep track of the process self.childProcess = subprocess.Popen(renderTaskCMD, stdout=log, stderr=log, **Utils.buildSubprocessArgs(False)) logger.info("Started PID %s to do Task %s", self.childProcess.pid, self.HydraTask.id) self.PSUtilProc = psutil.Process(self.childProcess.pid) #Wait for task to finish self.childProcess.communicate() #Get Exit Code, Record the results self.HydraTask.exitCode = self.childProcess.returncode if self.childProcess else 1234 logString = "\nProcess exited with code {0} at {1} on {2}\n" nowTime = datetime.datetime.now().replace(microsecond=0) log.write(logString.format(self.HydraTask.exitCode, nowTime, self.thisNode.host)) progressUpdateThread.terminate() #Update HydraTask and HydraJob with currentFrame, MPF, and RLTracker self.progressUpdate(commit=False) #EndTime self.HydraTask.endTime = datetime.datetime.now() #Work around for batch files if self.HydraJob.jobType == "BatchFile" and self.HydraTask.exitCode == 0: self.HydraTask.currentFrame = (self.HydraTask.endFrame + 1) self.HydraJob.renderLayerTracker = str((self.HydraTask.endFrame + 1)) #Status, Attempts. Failures if self.childKilled == 1: self.HydraTask.status = self.statusAfterDeath self.HydraTask.exitCode = 1 else: if self.HydraTask.exitCode == 0 and self.HydraTask.currentFrame >= originalCurrentFrame: status = FINISHED else: if self.HydraTask.exitCode == 0: log.write("\n\nERROR: Task returned exit code 0 but it appears to have not actually rendered any frames.") status = ERROR self.HydraJob.attempts += 1 if not self.HydraJob.failures or self.HydraJob.failures == "": self.HydraJob.failures = self.thisNode.host else: self.HydraJob.failures += ",{0}".format(self.thisNode.host) self.HydraTask.status = status #Update data on the DB with transaction() as t: self.HydraTask.update(t) self.HydraJob.update(t) self.resetThisNode() log.close() logger.info("Done with render task %s", self.HydraTask.id) self.childProcess = None self.PSUtilProc = None self.HydraJob = None self.HydraTask = None self.logPath = None