def softwareUpdater(): hydraPath = os.getenv("HYDRA") if not hydraPath: logger.error("HYDRA enviromental variable does not exit!") return False hydraPath, thisVersion = os.path.split(hydraPath) try: currentVersion = float(thisVersion.split("_")[-1]) except ValueError: logger.warning("Unable to obtain version number from file path. Assuming version number from Constants") currentVersion = Constants.VERSION versions = os.listdir(hydraPath) versions = [float(x.split("_")[-1]) for x in versions if x.startswith("dist_")] if not versions: return False highestVersion = max(versions) logger.debug("Comparing versions. Env: %s Latest: %s", currentVersion, highestVersion) if highestVersion > currentVersion: logger.info("Update found! Current Version is %s / New Version is %s", currentVersion, highestVersion) newPath = os.path.join(hydraPath, "dist_{}".format(highestVersion)) response = changeHydraEnviron(newPath) if not response: logger.critical("Could not update to newest environ for some reason!") return response else: return False
def getTotalRenderTime(self, returnDateTime=True): if not self.filteredLines: self.filterLines() if not self.filteredLines: return None reg = re.compile(r"total time for \d+ frames:.*") matches = [] for line in self.filteredLines: matches += reg.findall(line) if len(matches) > 1: logger.critical("More than one total time found in %s", self.fp) return None elif not matches: logger.debug("No total frame time found in %s", self.fp) return None line = matches[0].strip() h, m, s = self.getRsRenderTimeMatch(line) if h > 24: logger.critical("Critical! Log Parser is not setup to handle times longer than 24 hours yet...") return None totalTime = datetime.timedelta(hours=h, minutes=m, seconds=s) if returnDateTime: return totalTime else: return str(totalTime)
def filterLines(self): """Filters lines not realted to rendering so subsequent regexs run faster. Requires a filterRegex to be defined in the target class.""" reg = re.compile(self.filterRegex) lines = reg.findall(self.logFileContents) if not lines: logger.critical("Could not find any filtered lines in %s", self.fp) return None self.filteredLines = lines return lines
def changeHydraEnviron(newEnviron): if sys.platform == "win32": logger.info("Changing Hydra Environ to %s", newEnviron) proc = subprocess.Popen(["setx", "HYDRA", newEnviron], stdout=subprocess.PIPE) out, _ = proc.communicate() if out.find("SUCCESS") > 0: os.environ["HYDRA"] = newEnviron return True else: logger.critical("Could not change enviromental variable!") return False else: raise "Not Implemented!"
def main(): logger.info("Starting in %s", os.getcwd()) logger.info("arglist %s", sys.argv) #Check for other RenderNode isntances lockFile = InstanceLock("HydraRenderManager") lockStatus = lockFile.isLocked() logger.debug("Lock File Status: %s", lockStatus) if not lockStatus: logger.critical("Only one RenderManager is allowed to run at a time! Exiting...") sys.exit(-1) socketServer = RenderManagementServer() socketServer.createIdleLoop("Process_Render_Tasks_Thread", socketServer.processRenderTasks, interval=15)
def __init__(self): #Setup Class Variables self.renderThread = None self.childProcess = None self.PSUtilProc = None self.statusAfterDeath = None self.childKilled = 0 self.HydraJob = None self.HydraTask = None self.logPath = None #Get this node data from the database and make sure it exists self.thisNode = getThisNodeOBJ() logger.debug(self.thisNode) if not self.thisNode: logger.critical( "This node does not exist in the database! Please Register this node and try again." ) sys.exit(-1) return #Detect RedShift GPUs self.rsGPUs = Utils.getRedshiftPreference("SelectedCudaDevices") if self.rsGPUs: self.rsGPUs = self.rsGPUs.split(",")[:-1] self.rsGPUids = [x.split(":")[0] for x in self.rsGPUs] if len(self.rsGPUs) != len(self.rsGPUids): logger.warning("Problems parsing Redshift Preferences") logger.info("%s Redshift Enabled GPU(s) found on this node", len(self.rsGPUs)) logger.debug("GPUs available for rendering are %s", self.rsGPUs) else: logger.warning("Could not find available Redshift GPUs") #Create RenderLog Directory if it doesn't exit if not os.path.isdir(Constants.RENDERLOGDIR): os.makedirs(Constants.RENDERLOGDIR) self.unstickTask() self.thisNode.software_version = Constants.VERSION with transaction() as t: self.thisNode.update(t) #Run The Server port = int(Utils.getInfoFromCFG("network", "port")) self.startServerThread(port)
def getTotalFrameCount(self): if not self.filteredLines: self.filterLines() if not self.filteredLines: return None reg = re.compile(r"total time for (\d+) frames:.*") matches = [] for line in self.filteredLines: matches += reg.findall(line) if len(matches) > 1: logger.critical("More than one total time found in %s", self.fp) return None elif not matches: matches = [max(self.getSavedFrameNumbers())] return int(matches[0])
def createRenderJobs(allJobs, runningTasks): logger.debug("Creating Render Jobs") #Sorting sortedJobs = sorted(allJobs.values(), key=attrgetter("priority"), reverse=True) sortedJobs = sorted(sortedJobs, key=attrgetter("id")) #Filtering renderJobs = [] for job in sortedJobs: renderLayers = job.renderLayers.split(",") renderLayerTracker = [int(x) for x in job.renderLayerTracker.split(",")] if len(renderLayers) != len(renderLayerTracker): logger.critical("Malformed renderLayers or renderLayerTracker on job with id %d", job.id) with transaction() as t: t.cur.execute("UPDATE hydra_jobboard SET status = 'E' WHERE id = %s", (job.id,)) break runningRenderLayers = [x.renderLayer for x in runningTasks[int(job.id)]] if job.maxNodes > 0 and len(runningRenderLayers) > 0: if len(runningRenderLayers) >= int(job.maxNodes): logger.debug("Skipping job %d because it is over node limit", job.id) break if job.attempts >= job.maxAttempts: logger.debug("Skipping job %d because it is over attempt limit", job.id) with transaction() as t: t.cur.execute("UPDATE hydra_jobboard SET status = 'E' WHERE id = %s", (job.id,)) break for i in range(len(renderLayers)): if renderLayerTracker[i] <= job.endFrame: if renderLayers[i] not in runningRenderLayers: renderJobs.append([int(job.id), str(renderLayers[i])]) return renderJobs
hours = int(self.interval / 60 / 60) minutes = int(self.interval / 60 % 60) logger.info("Scheduler Sleeping for %d hours and %d minutes", hours, minutes) self.stopEvent.wait(self.interval) def pulse(): host = Utils.myHostName() with transaction() as t: t.cur.execute("UPDATE hydra_rendernode SET pulse = NOW() " "WHERE host = '{0}'".format(host)) if __name__ == "__main__": logger.info("Starting in %s", os.getcwd()) logger.info("arglist is %s", sys.argv) app = QApplication(sys.argv) app.quitOnLastWindowClosed = False lockFile = InstanceLock("HydraRenderNode") lockStatus = lockFile.isLocked() logger.debug("Lock File Status: %s", lockStatus) if not lockStatus: logger.critical("Only one RenderNode is allowed to run at a time! Exiting...") aboutBox(None, "ERROR", "Only one RenderNode is allowed to run at a time! Exiting...") sys.exit(-1) window = RenderNodeMainUI() retcode = app.exec_() lockFile.remove() sys.exit(retcode)