def softwareUpdater():
hydraPath = os.getenv("HYDRA")
if not hydraPath:
logger.error("HYDRA enviromental variable does not exit!")
return False
hydraPath, thisVersion = os.path.split(hydraPath)
try:
currentVersion = float(thisVersion.split("_")[-1])
except ValueError:
logger.warning("Unable to obtain version number from file path. Assuming version number from Constants")
currentVersion = Constants.VERSION
versions = os.listdir(hydraPath)
versions = [float(x.split("_")[-1]) for x in versions if x.startswith("dist_")]
if not versions:
return False
highestVersion = max(versions)
logger.debug("Comparing versions. Env: %s Latest: %s", currentVersion, highestVersion)
if highestVersion > currentVersion:
logger.info("Update found! Current Version is %s / New Version is %s", currentVersion, highestVersion)
newPath = os.path.join(hydraPath, "dist_{}".format(highestVersion))
response = changeHydraEnviron(newPath)
if not response:
logger.critical("Could not update to newest environ for some reason!")
return response
else:
return False
def getTotalRenderTime(self, returnDateTime=True):
if not self.filteredLines:
self.filterLines()
if not self.filteredLines:
return None
reg = re.compile(r"total time for \d+ frames:.*")
matches = []
for line in self.filteredLines:
matches += reg.findall(line)
if len(matches) > 1:
logger.critical("More than one total time found in %s", self.fp)
return None
elif not matches:
logger.debug("No total frame time found in %s", self.fp)
return None
line = matches[0].strip()
h, m, s = self.getRsRenderTimeMatch(line)
if h > 24:
logger.critical("Critical! Log Parser is not setup to handle times longer than 24 hours yet...")
return None
totalTime = datetime.timedelta(hours=h, minutes=m, seconds=s)
if returnDateTime:
return totalTime
else:
return str(totalTime)
def filterLines(self):
"""Filters lines not realted to rendering so subsequent regexs run faster.
Requires a filterRegex to be defined in the target class."""
reg = re.compile(self.filterRegex)
lines = reg.findall(self.logFileContents)
if not lines:
logger.critical("Could not find any filtered lines in %s", self.fp)
return None
self.filteredLines = lines
return lines
def changeHydraEnviron(newEnviron):
if sys.platform == "win32":
logger.info("Changing Hydra Environ to %s", newEnviron)
proc = subprocess.Popen(["setx", "HYDRA", newEnviron], stdout=subprocess.PIPE)
out, _ = proc.communicate()
if out.find("SUCCESS") > 0:
os.environ["HYDRA"] = newEnviron
return True
else:
logger.critical("Could not change enviromental variable!")
return False
else:
raise "Not Implemented!"
def main():
logger.info("Starting in %s", os.getcwd())
logger.info("arglist %s", sys.argv)
#Check for other RenderNode isntances
lockFile = InstanceLock("HydraRenderManager")
lockStatus = lockFile.isLocked()
logger.debug("Lock File Status: %s", lockStatus)
if not lockStatus:
logger.critical("Only one RenderManager is allowed to run at a time! Exiting...")
sys.exit(-1)
socketServer = RenderManagementServer()
socketServer.createIdleLoop("Process_Render_Tasks_Thread",
socketServer.processRenderTasks,
interval=15)
def __init__(self):
#Setup Class Variables
self.renderThread = None
self.childProcess = None
self.PSUtilProc = None
self.statusAfterDeath = None
self.childKilled = 0
self.HydraJob = None
self.HydraTask = None
self.logPath = None
#Get this node data from the database and make sure it exists
self.thisNode = getThisNodeOBJ()
logger.debug(self.thisNode)
if not self.thisNode:
logger.critical(
"This node does not exist in the database! Please Register this node and try again."
)
sys.exit(-1)
return
#Detect RedShift GPUs
self.rsGPUs = Utils.getRedshiftPreference("SelectedCudaDevices")
if self.rsGPUs:
self.rsGPUs = self.rsGPUs.split(",")[:-1]
self.rsGPUids = [x.split(":")[0] for x in self.rsGPUs]
if len(self.rsGPUs) != len(self.rsGPUids):
logger.warning("Problems parsing Redshift Preferences")
logger.info("%s Redshift Enabled GPU(s) found on this node",
len(self.rsGPUs))
logger.debug("GPUs available for rendering are %s", self.rsGPUs)
else:
logger.warning("Could not find available Redshift GPUs")
#Create RenderLog Directory if it doesn't exit
if not os.path.isdir(Constants.RENDERLOGDIR):
os.makedirs(Constants.RENDERLOGDIR)
self.unstickTask()
self.thisNode.software_version = Constants.VERSION
with transaction() as t:
self.thisNode.update(t)
#Run The Server
port = int(Utils.getInfoFromCFG("network", "port"))
self.startServerThread(port)
def getTotalFrameCount(self):
if not self.filteredLines:
self.filterLines()
if not self.filteredLines:
return None
reg = re.compile(r"total time for (\d+) frames:.*")
matches = []
for line in self.filteredLines:
matches += reg.findall(line)
if len(matches) > 1:
logger.critical("More than one total time found in %s", self.fp)
return None
elif not matches:
matches = [max(self.getSavedFrameNumbers())]
return int(matches[0])
def createRenderJobs(allJobs, runningTasks):
logger.debug("Creating Render Jobs")
#Sorting
sortedJobs = sorted(allJobs.values(), key=attrgetter("priority"), reverse=True)
sortedJobs = sorted(sortedJobs, key=attrgetter("id"))
#Filtering
renderJobs = []
for job in sortedJobs:
renderLayers = job.renderLayers.split(",")
renderLayerTracker = [int(x) for x in job.renderLayerTracker.split(",")]
if len(renderLayers) != len(renderLayerTracker):
logger.critical("Malformed renderLayers or renderLayerTracker on job with id %d", job.id)
with transaction() as t:
t.cur.execute("UPDATE hydra_jobboard SET status = 'E' WHERE id = %s", (job.id,))
break
runningRenderLayers = [x.renderLayer for x in runningTasks[int(job.id)]]
if job.maxNodes > 0 and len(runningRenderLayers) > 0:
if len(runningRenderLayers) >= int(job.maxNodes):
logger.debug("Skipping job %d because it is over node limit", job.id)
break
if job.attempts >= job.maxAttempts:
logger.debug("Skipping job %d because it is over attempt limit", job.id)
with transaction() as t:
t.cur.execute("UPDATE hydra_jobboard SET status = 'E' WHERE id = %s", (job.id,))
break
for i in range(len(renderLayers)):
if renderLayerTracker[i] <= job.endFrame:
if renderLayers[i] not in runningRenderLayers:
renderJobs.append([int(job.id), str(renderLayers[i])])
return renderJobs
hours = int(self.interval / 60 / 60)
minutes = int(self.interval / 60 % 60)
logger.info("Scheduler Sleeping for %d hours and %d minutes", hours, minutes)
self.stopEvent.wait(self.interval)
def pulse():
host = Utils.myHostName()
with transaction() as t:
t.cur.execute("UPDATE hydra_rendernode SET pulse = NOW() "
"WHERE host = '{0}'".format(host))
if __name__ == "__main__":
logger.info("Starting in %s", os.getcwd())
logger.info("arglist is %s", sys.argv)
app = QApplication(sys.argv)
app.quitOnLastWindowClosed = False
lockFile = InstanceLock("HydraRenderNode")
lockStatus = lockFile.isLocked()
logger.debug("Lock File Status: %s", lockStatus)
if not lockStatus:
logger.critical("Only one RenderNode is allowed to run at a time! Exiting...")
aboutBox(None, "ERROR", "Only one RenderNode is allowed to run at a time! Exiting...")
sys.exit(-1)
window = RenderNodeMainUI()
retcode = app.exec_()
lockFile.remove()
sys.exit(retcode)
