def getDatabaseInfo():
logger.debug("Finding login information...")
#Get databse information
host = Utils.getInfoFromCFG("database", "host")
domain = Utils.getInfoFromCFG("network", "dnsDomainExtension").replace(" ", "")
if domain != "" and host != "localhost":
host += ".{}".format(domain)
databaseName = Utils.getInfoFromCFG("database", "db")
port = int(Utils.getInfoFromCFG("database", "port"))
db_username = Utils.getInfoFromCFG("database", "username")
#Get login information
autoLogin = Utils.getInfoFromCFG("database", "autologin")
autoLogin = True if str(autoLogin).lower()[0] == "t" else False
if autoLogin:
_db_password = PasswordStorage.loadCredentials(db_username)
if not _db_password:
autoLogin = False
if not autoLogin:
returnValues = PasswordStorage.qtPrompt()
if not returnValues[0] or not returnValues[1]:
logger.error("Could not login!")
sys.exit(1)
else:
db_username = returnValues[0]
_db_password = returnValues[1]
return host, db_username, _db_password, databaseName, port
def taskActionHandler(self, mode):
"""A catch-all function for performing actions on the items selected
in the taskTree"""
taskIDs = self.getTaskTreeSel("IDs")
if not taskIDs:
return None
if mode == "kill":
response = yesNoBox(self, "Confirm", "Are you sure you want to kill these tasks: {}".format(taskIDs))
if response == QMessageBox.No:
return None
cols = ["id", "status", "exitCode", "endTime", "host"]
taskOBJs = [hydra_taskboard.fetch("WHERE id = %s", (t,), cols=cols)
for t in taskIDs]
responses = [task.kill() for task in taskOBJs]
if not all(responses):
failureIDXes = [i for i, x in enumerate(responses) if not x]
failureIDs = [taskIDs[i] for i in failureIDXes]
logger.error("Task Kill failed on %s", failureIDs)
warningBox(self, "Task Kill Error!",
"Task Kill failed on task(s) with IDs {}".format(failureIDs))
elif mode == "log":
if len(taskIDs) > 1:
choice = yesNoBox(self, "Confirm", "You have {0} tasks selected. Are you sure you want to open {0} logs?".format(len(taskIDs)))
if choice == QMessageBox.No:
return None
map(self.openLogFile, taskIDs)
elif mode == "data":
if len(taskIDs) == 1:
self.revealDataTable(taskIDs, hydra_taskboard, "WHERE id = %s")
else:
self.revealDetailedHandler(taskIDs, hydra_taskboard, "WHERE id = %s")
def getOffThisNodeHandler(self):
response = self.thisNode.getOff()
self.updateThisNodeInfo()
if response:
logger.info("Node Offlined and Task Killed")
else:
logger.error("Node could not be Onlined or the Task could not be killed!")
def offlineThisNodeHandler(self):
response = self.thisNode.offline()
self.updateThisNodeInfo()
if response:
logger.info("Node Offlined")
else:
logger.error("Node could not be Offlined!")
def getAnswer(self, question):
"""Send the question to a remote server and get an answer back"""
#Create a TCP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
#Connect to the server
logger.debug("TCP Connect to %s %d", self.hostname, self.port)
if self.hostname is None:
return None
try:
sock.connect((self.hostname, self.port))
#Convert the question to ASCII
questionBytes = pickle.dumps(question)
#Send the question
sock.sendall(questionBytes)
#Close the sending half of the connection so the other side
#Knows we're done sending
sock.shutdown(socket.SHUT_WR)
#Read the response, an ASCII encoded object
answerBytes = sock.recv(Constants.MANYBYTES)
#Convert the response to an object
try:
answer = pickle.loads(answerBytes)
except EOFError:
logger.error("EOF Error on Connections.TCPConnection.getAnswer()")
logger.error("answerBytes = %s", str(answerBytes))
answer = None
except socket.error as err:
logger.debug(err)
answer = None
finally:
sock.close()
logger.debug("TCP Connection to %s %d was closed.", self.hostname, self.port)
return answer
def softwareUpdater():
hydraPath = os.getenv("HYDRA")
if not hydraPath:
logger.error("HYDRA enviromental variable does not exit!")
return False
hydraPath, thisVersion = os.path.split(hydraPath)
try:
currentVersion = float(thisVersion.split("_")[-1])
except ValueError:
logger.warning("Unable to obtain version number from file path. Assuming version number from Constants")
currentVersion = Constants.VERSION
versions = os.listdir(hydraPath)
versions = [float(x.split("_")[-1]) for x in versions if x.startswith("dist_")]
if not versions:
return False
highestVersion = max(versions)
logger.debug("Comparing versions. Env: %s Latest: %s", currentVersion, highestVersion)
if highestVersion > currentVersion:
logger.info("Update found! Current Version is %s / New Version is %s", currentVersion, highestVersion)
newPath = os.path.join(hydraPath, "dist_{}".format(highestVersion))
response = changeHydraEnviron(newPath)
if not response:
logger.critical("Could not update to newest environ for some reason!")
return response
else:
return False
def __exit__(self, errorType, traceback, value):
if errorType is None:
#logger.debug("commit %s", self)
self.cur.execute("commit")
else:
logger.error("rollback %s", self)
self.cur.execute("rollback")
#logger.debug("exit transaction %s", self)
self.db.close()
def assignTask(node, task, job):
logger.debug("Assigning task with id %d to node %s", task.id, node.host)
connection = TCPConnection(hostname=node.host)
response = connection.getAnswer(StartRenderQuestion(job, task))
if response:
logger.debug("Task %d was accepted on %s", task.id, node.host)
else:
logger.error("Task %d was declined on %s", task.id, node.host)
return response
def updateAttr(self, attr, value):
if attr not in self.attributes():
logger.error("Attr %s not found in %s", attr, self.tableName())
return
with transaction() as t:
updateStr = "UPDATE {0} SET {1} = %s WHERE {2} = %s"
t.cur.execute(updateStr.format(self.tableName(), attr, self.primaryKey),
(value, getattr(self, self.primaryKey)))
self.__dict__[attr] = value
return True
def setupGlobals(self):
#Scene file should be first sys.argv
try:
self.scene = sys.argv[1]
self.scene = self.scene.replace('\\', '/')
except IndexError:
self.scene = ""
#Get the -flag args
try:
opts = getopt.getopt(sys.argv[2:], "s:e:n:p:l:x:m:d:c:q:t:")[0]
except getopt.GetoptError:
logger.error("Bad Opt!")
aboutBox(self, "Bad Opt!", "One of the command line options you entered was invalid.\n"+
"\nPlease remove any unkown opts and try again.")
sys.exit(2)
#Defaults
defName = self.scene.split("/")[-1]
self.settingsDict = {"-s":101, #Start Frame (Int)
"-e":101, #End Frame (Int)
"-n":defName, #Nice Name (Str)
"-p":"", #Proj (Str)
"-l":"", #Render Layers (Str,Sep,By,Comma)
"-x":"", #Executabe (Str)
"-m":"", #CMD (Str)
"-d":"", #RenderDirectory (Str)
"-c":"", #Compatabilities (Str,Sep,By,Comma)
"-q":"", #Project Name (Str)
"-t":"", #Job Type (Str)
}
#Apply the -flag args
optsDict = dict(opts)
keys = list(optsDict.keys())
for key in keys:
self.settingsDict[key] = optsDict[key]
logger.debug("Setting Key '%s' with opt: '%s'", key, str(optsDict[key]))
#Fix paths
self.settingsDict["-p"] = self.settingsDict["-p"].replace('\\', '/')
#Fix Compatabilities
self.settingsDict["-c"] = self.settingsDict["-c"].split(",")
#Add underscores to niceName
self.settingsDict["-n"] = self.settingsDict["-n"].replace(" ", "_")
#Move RenderDir to Base CMD
if self.settingsDict["-d"] != "":
self.settingsDict["-d"] = self.settingsDict["-d"].replace('\\', '/')
self.settingsDict["-m"] += " -rd \"{0}\"".format(self.settingsDict["-d"])
def getOffThisNodeHandler(self):
"""Offlines the node and sends a message to the render node server
running on localhost to kill its current task(task will be
resubmitted)"""
thisNode = NodeUtils.getThisNodeOBJ()
if not thisNode:
return
choice = yesNoBox(self, "Confirm", Constants.GETOFFLOCAL_STRING)
if choice == QMessageBox.Yes:
response = thisNode.getOff()
if not response:
logger.error("Could not GetOff this node!")
warningBox(self, "GetOff Error", "Could not GetOff this node!")
self.populateNodeTree()
self.updateStatusBar(thisNode)
def __init__(self):
QMainWindow.__init__(self)
self.setupUi(self)
with open(Utils.findResource("styleSheet.css"), "r") as myStyles:
self.setStyleSheet(myStyles.read())
self.thisNode = NodeUtils.getThisNodeOBJ()
self.isVisable = True
self.pulseThreadStatus = False
self.renderServerStatus = False
self.schedThreadStatus = False
self.autoUpdateStatus = False
if not self.thisNode:
self.offlineButton.setEnabled(False)
self.getoffButton.setEnabled(False)
logger.error("Node does not exist in database!")
aboutBox(self, "Error",
"This node was not found in the database! If you wish to render "
"on this node it must be registered with the databse. Run "
"Register.exe or Register.py to regiester this node and "
" try again.")
sys.exit(1)
self.currentSchedule = self.thisNode.weekSchedule
self.currentScheduleEnabled = self.thisNode.scheduleEnabled
self.buildUI()
self.connectButtons()
self.updateThisNodeInfo()
self.startupServers()
logger.info("Render Node Main is live! Waiting for tasks...")
try:
autoHide = True if str(sys.argv[1]).lower() == "true" else False
logger.info(autoHide)
except IndexError:
autoHide = False
if autoHide and self.trayIconBool:
logger.info("Autohide is enabled!")
self.sendToTrayHandler()
else:
self.show()
def findNextEvent(now, dbData):
"""Take the current datetime and the decoded schedule data from the DB and
find the next scheduling event"""
nowDayOfWeek = now.isoweekday()
nowTime = now.time()
dataList = dbData.split(",")
if len(dataList) < 2:
return None
dataDict = {}
for actionItem in dataList:
actionItemList = actionItem.split("-")
dayOfWeek = int(actionItemList[0])
action = int(actionItemList[2])
timeList = [int(t) for t in actionItemList[1].split(":")]
timeObject = datetime.time(timeList[0], timeList[1])
try:
dataDict[dayOfWeek] += [[timeObject, action]]
except KeyError:
dataDict[dayOfWeek] = [[timeObject, action]]
#scheule is a nested list like [[time, action], [time,action]]
todaySchedule, newDayOfWeek = findSchedule(nowDayOfWeek, dataDict)
if not todaySchedule:
return None
sched = None
#Check each schedule item's activation time, if one of them is after now
# then this schedule will work for today
for schedItem in todaySchedule:
if schedItem[0] > nowTime:
logger.debug("Schedule Found: %s", sched)
sched = schedItem
#If not then the next schdule item is probably on a date later in the week.
#Iterate the day of week and look again.
if not sched:
newDayOfWeek += 1
todaySchedule, newDayOfWeek = findSchedule(newDayOfWeek, dataDict)
sched = todaySchedule[0]
if not sched:
logger.error("Could not find schedule")
return None
return [newDayOfWeek] + sched
def getRedshiftPreference(attribute):
"""Return an attribute from the Redshift preferences.xml file"""
if sys.platform == "win32":
try:
tree = ET.parse("C:\\ProgramData\\Redshift\\preferences.xml")
except IOError:
logger.error("Could not find Redshift Preferences!")
return None
else:
#TODO:Other platforms
return None
root = tree.getroot()
perfDict = {c.attrib["name"]:c.attrib["value"] for c in root}
try:
return perfDict[attribute]
except KeyError:
logger.error("Could not find %s in Redshift Preferences!", attribute)
return None
def sendQuestion(self, question):
"""Send question without waiting for a response"""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
returnVal = False
try:
#Connect to the server
logger.debug("TCP Connect to %s %d", self.hostname, self.port)
if self.hostname is None:
return None
sock.connect((self.hostname, self.port))
questionBytes = pickle.dumps(question)
sock.sendall(questionBytes)
sock.shutdown(socket.SHUT_WR)
sock.close()
returnVal = True
except socket.error as err:
logger.error(err)
logger.debug("TCP Connection to %s %d was closed.", self.hostname, self.port)
return returnVal
def loginButtonHandler(self):
self.db_username = str(self.user.text())
self._db_password = str(self.password.text())
if self.remoteAccess.isChecked():
#self.host = "REMOTE"
pass
try:
MySQLdb.connect(host=self.host, user=self.db_username,
passwd=self._db_password, db=self.databaseName,
port=self.port)
self.loginSuccess = True
self.close()
except MySQLdb.Error:
logger.error("Could not login!")
aboutBox(self, "Could Not Login",
"Invalid username/password or server is down...")
def createTaskCMD(self, hydraJob, platform="win32"):
execs = hydra_executable.fetch(multiReturn=True)
if platform == "win32":
execsDict = {ex.name: ex.win32 for ex in execs}
else:
execsDict = {ex.name: ex.linux for ex in execs}
#Not sure if Maya for Linux or Maya 2016 thing but one of the two is
# is appending quotes on the file cmd and messing everything up
taskFile = os.path.abspath(hydraJob.taskFile)
if platform == "win32":
taskFile = "\"{0}\"".format(taskFile)
else:
taskFile = hydraJob.taskFile
baseCMD = shlex.split(hydraJob.baseCMD)
if hydraJob.jobType in ["RedshiftRender", "MentalRayRender"]:
renderList = [execsDict[hydraJob.execName]]
renderList += baseCMD
renderList += ["-s", self.currentFrame, "-e", self.endFrame, "-b",
hydraJob.byFrame, "-rl", hydraJob.renderLayers,
taskFile]
elif hydraJob.jobType == "FusionComp":
renderList = [execsDict[hydraJob.execName], taskFile]
renderList += baseCMD
renderList += ["/render", "/quiet", "/frames",
"{0}..{1}".format(self.startFrame, self.endFrame),
"/by", hydraJob.byFrame, "/exit", "/log TestLog.txt", "/verbose"]
elif hydraJob.jobType == "BatchFile":
renderList = [taskFile, hydraJob.baseCMD]
else:
logger.error("Bad Job Type!")
return None
if hydraJob.jobType == "BatchFile":
return " ".join([str(x) for x in renderList])
else:
return [str(x) for x in renderList]
def remove(self):
if not self.locked:
return
if sys.platform == "win32":
if hasattr(self, "tempFile"):
try:
os.close(self.tempFile)
os.unlink(self.tempFilePath)
except Exception as e:
logger.error(e)
else:
logger.warning("No temp file found for %s", self.name)
else:
try:
fnctl.lockf(self.tempFile, fcntl.LOCK_UN)
if os.path.isfile(self.tempFilePath):
os.unlink(self.tempFilePath)
except Exception as e:
logger.error(e)
def killCurrentJob(self, statusAfterDeath):
"""Kills the render node's current job if it's running one.
Return Codes: 1 = process killed, -1 = parent could not be killed,
-9 = child could not be killed, -10 = child and parent could not be killed"""
self.statusAfterDeath = statusAfterDeath
self.childKilled = 1
if not self.childProcess or not self.PSUtilProc:
logger.info("No task is running!")
return
#Gather subprocesses just in case
if self.PSUtilProc.is_running():
childrenProcs = self.PSUtilProc.children(recursive=True)
else:
logger.info(
"PID '%s' could not be found! Task is probably already dead.",
self.childProcess.pid)
return
#Try to kill the main process
#terminate() = SIGTERM, kill() = SIGKILL
logger.info("Killing main task with PID %s", self.PSUtilProc.pid)
self.PSUtilProc.terminate()
_, alive = psutil.wait_procs([self.PSUtilProc], timeout=15)
if len(alive) > 0:
self.PSUtilProc.kill()
_, alive = psutil.wait_procs([self.PSUtilProc], timeout=15)
if len(alive) > 0:
logger.error("Could not kill PID %s", self.PSUtilProc.pid)
self.childKilled = -1
#Try to kill the children if they are still running
_ = [proc.terminate() for proc in childrenProcs if proc.is_running()]
_, alive = psutil.wait_procs(childrenProcs, timeout=15)
if len(alive) > 0:
_ = [proc.kill() for proc in alive]
_, alive = psutil.wait_procs(alive, timeout=15)
if len(alive) > 0:
#ADD negative 10 to the return code
self.childKilled += -10
def schedulerMain(self):
if not self.thisNode:
self.scheduleThreadPixmap.setPixmap(self.needsAttentionPixmap)
logger.error("Node OBJ not found by schedulerMain! Checking again in 24 hours.")
#Sleep for 24 hours
return 86400
self.updateThisNodeInfo()
sleepTime, nowStatus = NodeUtils.calcuateSleepTimeFromNode(self.thisNode.host)
if not sleepTime or not nowStatus:
logger.error("Could not find schdule! Checking again in 24 hours.")
return 86400
if nowStatus == READY:
self.startupEvent()
else:
self.shutdownEvent()
#Add an extra minute just in case
return sleepTime + 60
def __init__(self, name):
self.locked = False
self.name = name
self.tempFilePath = os.path.join(BASEDIR, "{}.lock".format(self.name))
self.tempFilePath = os.path.abspath(self.tempFilePath)
logger.info("Temp File: %s", self.tempFilePath)
#Windows
if sys.platform == "win32":
try:
if os.path.exists(self.tempFilePath):
os.unlink(self.tempFilePath)
logger.debug("Unlink %s", self.tempFilePath)
self.tempFile = os.open(self.tempFilePath, os.O_CREAT | os.O_EXCL | os.O_RDWR)
self.locked = True
except Exception as e:
if e.errno == 13:
logger.error("Another Instance of %s is already running!", self.name)
else:
logger.error(e)
#Linux
else:
import fcntl
self.tempFile = open(self.tempFilePath, "w")
self.tempFile.flush()
try:
fcntl.lockf(self.tempFile, fcntl.LOCK_EX | fcntl.LOCK_NB)
self.locked = True
except IOError:
logger.error("Another Instance of %s is already running", self.name)
def findSchedule(dayOfWeek, dataDict):
"""Take and ISOWeekday and a dictionary from within findNextEvent and finds
the next event during the given ISOWeekday and in the dataDict."""
schedule = None
i = 0
while not schedule and i < 9:
#Check if day of week is the the dataDict
if dayOfWeek in dataDict.keys():
schedule = dataDict[dayOfWeek]
#If not add one to dayOfWeek unless day of week is 6 (Saturday),
# then reset to 0 (Sunday)
else:
if dayOfWeek < 6:
dayOfWeek += 1
else:
dayOfWeek = 0
#Finally, increase counter
i += 1
if i > 8:
logger.error("Can't find next event!")
return schedule, dayOfWeek
def launchHydraApp(app, wait=0):
"""Primarily for killing the app and restarting it"""
hydraPath = os.getenv("HYDRA")
if not hydraPath:
logger.error("HYDRA enviromental variable does not exit!")
return None
if sys.platform == "win32":
execs = os.listdir(hydraPath)
if not any([x.startswith(app) for x in execs]):
logger.error("%s is not a vaild Hydra Win32 App", app)
return None
distFolder, _ = os.path.split(hydraPath)
shortcutPath = os.path.join(distFolder, "_shortcuts")
ext = ".bat" if sys.platform == "win32" else ".sh"
script = "StartHydraApp{}".format(ext)
scriptPath = os.path.join(shortcutPath, script)
command = [scriptPath, app]
if wait > 0:
command += [str(int(wait))]
subprocess.Popen(command, stdout=False)
def reset(self, resetData):
if not resetData:
logger.debug("No reset data recieved")
return 0
resetRLs = resetData[0]
currentFrame = resetData[1]
nodeReset = resetData[2]
responses = []
if nodeReset:
responses.append(self.updateAttr("failures", ""))
responses.append(self.updateAttr("attempts", 0))
if resetRLs:
if currentFrame > self.endFrame:
logger.error("New start frame is higher than the end frame! Aboring!")
return -1
if currentFrame < self.startFrame:
logger.warning("New start frame is lower than original start frame, resetting to default.")
currentFrame = 0
if currentFrame == self.startFrame:
currentFrame = 0
idxList = [self.renderLayers.split(",").index(x) for x in resetRLs]
rlTracker = self.renderLayerTracker.split(",")
for i in idxList:
rlTracker[i] = str(currentFrame)
responses.append(self.updateAttr("renderLayerTracker", ",".join(rlTracker)))
if self.status in [KILLED, FINISHED]:
responses.append(self.updateAttr("status", PAUSED))
return 0 if all(responses) else -2
def getOffRenderNodesHandler(self):
"""Kills and Offlines each node selected in the nodeTree"""
hosts = self.getNodeTreeSel()
if not hosts:
return
choice = yesNoBox(self, "Confirm", Constants.GETOFF_STRING + str(hosts))
if choice == QMessageBox.No:
return
#else
cols = ["host", "status", "task_id"]
renderHosts = [hydra_rendernode.fetch("WHERE host = %s", (host,), cols=cols)
for host in hosts]
responses = [host.getOff() for host in renderHosts]
if not all(responses):
failureIDXes = [i for i, x in enumerate(responses) if not x]
failureHosts = [renderHosts[i] for i in failureIDXes]
logger.error("Could not get off %s", failureHosts)
warningBox(self, "GetOff Error!",
"Could not get off the following hosts: {}".format(failureHosts))
self.populateNodeTree()
def launchRenderTask(self, HydraJob, HydraTask):
"""Does the actual rendering, then records the results on the database"""
logger.info("Starting task with id %s on job with id %s", HydraTask.id, HydraJob.id)
self.HydraJob = HydraJob
self.HydraTask = HydraTask
self.childKilled = 0
self.statusAfterDeath = None
self.childProcess = None
self.PSUtilProc = None
originalCurrentFrame = int(self.HydraTask.currentFrame)
renderTaskCMD = self.HydraTask.createTaskCMD(self.HydraJob, sys.platform)
logger.debug(renderTaskCMD)
self.logPath = self.HydraTask.getLogPath()
logger.info("Starting render task %s", self.HydraTask.id)
try:
log = file(self.logPath, 'w')
except (IOError, OSError, WindowsError) as e:
logger.error(e)
self.thisNode.getOff()
return
log.write('Hydra log file {0} on {1}\n'.format(self.logPath, self.HydraTask.host))
log.write('RenderNode is {0}\n'.format(sys.argv))
log.write('Command: {0}\n\n'.format(renderTaskCMD))
Utils.flushOut(log)
progressUpdateThread = stoppableThread(self.progressUpdate, 300,
"Progress_Update_Thread")
#Run the job and keep track of the process
self.childProcess = subprocess.Popen(renderTaskCMD,
stdout=log, stderr=log,
**Utils.buildSubprocessArgs(False))
logger.info("Started PID %s to do Task %s", self.childProcess.pid, self.HydraTask.id)
self.PSUtilProc = psutil.Process(self.childProcess.pid)
#Wait for task to finish
self.childProcess.communicate()
#Get Exit Code, Record the results
self.HydraTask.exitCode = self.childProcess.returncode if self.childProcess else 1234
logString = "\nProcess exited with code {0} at {1} on {2}\n"
nowTime = datetime.datetime.now().replace(microsecond=0)
log.write(logString.format(self.HydraTask.exitCode, nowTime,
self.thisNode.host))
progressUpdateThread.terminate()
#Update HydraTask and HydraJob with currentFrame, MPF, and RLTracker
self.progressUpdate(commit=False)
#EndTime
self.HydraTask.endTime = datetime.datetime.now()
#Work around for batch files
if self.HydraJob.jobType == "BatchFile" and self.HydraTask.exitCode == 0:
self.HydraTask.currentFrame = (self.HydraTask.endFrame + 1)
self.HydraJob.renderLayerTracker = str((self.HydraTask.endFrame + 1))
#Status, Attempts. Failures
if self.childKilled == 1:
self.HydraTask.status = self.statusAfterDeath
self.HydraTask.exitCode = 1
else:
if self.HydraTask.exitCode == 0 and self.HydraTask.currentFrame >= originalCurrentFrame:
status = FINISHED
else:
if self.HydraTask.exitCode == 0:
log.write("\n\nERROR: Task returned exit code 0 but it appears to have not actually rendered any frames.")
status = ERROR
self.HydraJob.attempts += 1
if not self.HydraJob.failures or self.HydraJob.failures == "":
self.HydraJob.failures = self.thisNode.host
else:
self.HydraJob.failures += ",{0}".format(self.thisNode.host)
self.HydraTask.status = status
#Update data on the DB
with transaction() as t:
self.HydraTask.update(t)
self.HydraJob.update(t)
self.resetThisNode()
log.close()
logger.info("Done with render task %s", self.HydraTask.id)
self.childProcess = None
self.PSUtilProc = None
self.HydraJob = None
self.HydraTask = None
self.logPath = None
def buildUI(self):
def addItem(name, handler, statusTip, menu):
action = QAction(name, self)
action.setStatusTip(statusTip)
action.triggered.connect(handler)
menu.addAction(action)
#Add Logging handlers for output field
emStream = EmittingStream(textWritten=self.normalOutputWritten)
handler = logging.StreamHandler(emStream)
handler.setLevel(logging.INFO)
handler.setFormatter(outputWindowFormatter)
logger.addHandler(handler)
sys.stdout = EmittingStream(textWritten=self.normalOutputWritten)
sys.stderr = EmittingStream(textWritten=self.normalOutputWritten)
#Get Pixmaps and Icon
self.donePixmap = QPixmap(Utils.findResource("Images/status/done.png"))
self.inProgPixmap = QPixmap(Utils.findResource("Images/status/inProgress.png"))
self.needsAttentionPixmap = QPixmap(Utils.findResource("Images/status/needsAttention.png"))
self.nonePixmap = QPixmap(Utils.findResource("Images/status/none.png"))
self.notStartedPixmap = QPixmap(Utils.findResource("Images/status/notStarted.png"))
self.refreshPixmap = QPixmap(Utils.findResource("Images/refresh.png"))
self.refreshIcon = QIcon()
self.refreshIcon.addPixmap(self.refreshPixmap)
self.RIcon = QIcon(Utils.findResource("Images/RenderNodeMain.png"))
self.isVisable = True
self.refreshButton.setIcon(self.refreshIcon)
self.renderServerPixmap.setPixmap(self.notStartedPixmap)
self.scheduleThreadPixmap.setPixmap(self.notStartedPixmap)
self.pulseThreadPixmap.setPixmap(self.notStartedPixmap)
self.setWindowIcon(self.RIcon)
#Setup tray icon
self.trayIcon = QSystemTrayIcon()
self.trayIconBool = self.trayIcon.isSystemTrayAvailable()
if self.trayIconBool:
self.trayIcon.setIcon(self.RIcon)
self.trayIcon.show()
self.trayIcon.setVisible(True)
self.trayIcon.activated.connect(self.activate)
self.trayIcon.messageClicked.connect(self.activate)
#Tray Icon Context Menu
self.taskIconMenu = QMenu(self)
addItem("Open", self.showWindowHandler,
"Show the RenderNodeMain Window", self.taskIconMenu)
self.taskIconMenu.addSeparator()
addItem("Update", self.updateThisNodeInfo,
"Fetch the latest information from the Database", self.taskIconMenu)
self.taskIconMenu.addSeparator()
addItem("Online", self.onlineThisNodeHandler,
"Online this node", self.taskIconMenu)
addItem("Offline", self.offlineThisNodeHandler,
"Offline this node", self.taskIconMenu)
addItem("GetOff!", self.getOffThisNodeHandler,
"Kill the current task and offline this node", self.taskIconMenu)
self.trayIcon.setContextMenu(self.taskIconMenu)
else:
logger.error("Tray Icon Error! Could not create tray icon.")
aboutBox(self, "Tray Icon Error",
"Could not create tray icon. Minimizing to tray has been disabled.")
self.trayButton.setEnabled(False)
"""Registers a node with the database."""
#Standard
import os
import sys
#Third Party
#pylint: disable=E0611
from MySQLdb import IntegrityError
#Hydra
from Setups.LoggingSetup import logger
from Setups.MySQLSetup import hydra_rendernode, OFFLINE, transaction
import Utilities.Utils as Utils
if __name__ == "__main__":
me = Utils.myHostName()
hydraPath, execFile = os.path.split(sys.argv[0])
logger.info(hydraPath)
response = Utils.changeHydraEnviron(hydraPath)
if response:
try:
with transaction() as t:
hydra_rendernode(host=me, status=OFFLINE, minPriority=0).insert(t)
except IntegrityError:
logger.info("Host %s already exists in the hydra_rendernode table on the databse", me)
else:
logger.error("Could not set Hydra Environ! No changes where made. Exiting...")
raw_input("\nPress enter to exit...")
def loadTaskTree(self, job_id, clear=False):
"""Loads all subtasks into the taskTree given a job id. Clear will
clear the widget before loading the data."""
if clear:
self.taskTree.clear()
#SetupTrunks
job = hydra_jobboard.fetch("WHERE id = %s", (job_id,),
cols=["id", "renderLayers",
"renderLayerTracker", "startFrame",
"endFrame", "status", "jobType"])
for rl in job.renderLayers.split(","):
rootSearch = self.taskTree.findItems(str(rl), Qt.MatchExactly, 0)
if rootSearch:
root = rootSearch[0]
else:
root = QTreeWidgetItem(self.taskTree, [rl])
root.setFont(0, QFont('Segoe UI', 10, QFont.DemiBold))
#Add tasks to taskTree
tasks = hydra_taskboard.fetch("WHERE job_id = %s", (job_id,),
multiReturn=True,
cols=["id", "renderLayer", "status",
"startTime", "endTime", "host",
"startFrame", "endFrame",
"currentFrame", "exitCode"])
self.setNodeTaskColors(tasks)
for task in tasks:
rootSearch = self.taskTree.findItems(str(task.renderLayer), Qt.MatchExactly, 0)
if rootSearch:
root = rootSearch[0]
taskData = self.formatTaskData(task)
taskSearch = self.taskTree.findItems(str(task.id), Qt.MatchRecursive, 1)
if taskSearch:
taskItem = taskSearch[0]
for i in range(0, 9):
taskItem.setData(i, 0, taskData[i])
else:
taskItem = QTreeWidgetItem(root, taskData)
#Formatting
taskItem.setBackgroundColor(2, niceColors[task.status])
if task.host == self.thisNodeName:
taskItem.setFont(3, QFont('Segoe UI', 8, QFont.DemiBold))
else:
logger.error("Could not find root for renderLayer %s on task %s!", task.renderLayer, task.id)
#Add default data if active tasks don't exist
#Do formatting on all tasks
#Note that since the RLs were added in order we can use the same index
rlTracker = [int(x) for x in job.renderLayerTracker.split(",")]
for i in range(0, self.taskTree.topLevelItemCount()):
topLevelItem = self.taskTree.topLevelItem(i)
if topLevelItem.childCount() < 1:
defaultTaskData = [topLevelItem.text(0), "None", niceNames[READY],
"None", str(job.startFrame),
str(job.endFrame), str(rlTracker[i]),
"None", "None", "None", "None"]
QTreeWidgetItem(topLevelItem, defaultTaskData)
#Add top level formatting to taskTree data
if rlTracker[i] >= job.endFrame:
topLevelItem.setBackgroundColor(0, niceColors[FINISHED])
else:
statusToUse = topLevelItem.child(topLevelItem.childCount() - 1).text(2)
statusToUse = "Ready" if statusToUse == "Finished" else statusToUse
topLevelItem.setBackgroundColor(0, niceColors[niceNamesRev[str(statusToUse)]])
topLevelItem.setExpanded(True)
def jobActionHandler(self, mode):
"""A catch-all function for performing actions on the items selected
in the jobTree"""
#pylint: disable=R0912
jobIDs = self.getJobTreeSel()
if not jobIDs:
return None
cols = ["id", "status", "renderLayers", "renderLayerTracker", "archived",
"priority", "startFrame", "endFrame", "failures", "attempts"]
jobOBJs = [hydra_jobboard.fetch("WHERE id = %s", (jID,), cols=cols) for jID in jobIDs]
#Start Job
if mode == "start":
[job.start() for job in jobOBJs]
self.populateJobTree()
#Pause Job
elif mode == "pause":
[job.pause() for job in jobOBJs]
self.populateJobTree()
#Reveal Detailed Data
elif mode == "data":
if len(jobIDs) == 1:
self.revealDataTable(jobIDs, hydra_jobboard, "WHERE id = %s")
else:
self.revealDetailedHandler(jobIDs, hydra_jobboard, "WHERE id = %s")
#Kill
elif mode == "kill":
choice = yesNoBox(self, "Confirm", "Really kill the selected jobs?")
if choice == QMessageBox.No:
return None
rawResponses = [job.kill() for job in jobOBJs]
responses = [all(res) for res in rawResponses]
self.populateJobTree()
respString = "Job Kill returned the following errors:\n"
if not all(responses):
failureIDXes = [i for i, x in enumerate(responses) if not x]
for idx in failureIDXes:
taskString = "\t"
taskFailures = [i for i, x in enumerate(rawResponses[idx]) if not x]
statusSuccess = taskFailures[-1]
taskSuccess = taskFailures[:-1]
taskString += "Job '{}' had ".format(jobIDs[i])
if not statusSuccess:
taskString += "an error changing its status and "
taskString += "{} errors killing subtasks.\n".format(len(taskSuccess))
respString += taskString
logger.error(respString)
warningBox(self, "Job Kill Errors!", respString)
#Reset
elif mode == "reset":
if len(jobOBJs) > 1:
choice = yesNoBox(self, "Confirm", "Really reset the selected jobs?\nNote that this will open a dialog for EACH selected job to be reset.")
if choice == QMessageBox.No:
return None
errList = []
for job in jobOBJs:
if job.status in [KILLED, PAUSED, READY, FINISHED]:
data = ResetDialog.create([job.renderLayers.split(","),
job.startFrame])
response = job.reset(data)
else:
aboutBox(self, "Warning", "Job {} could not be reset because it is running. Please kill/pause it and try again.".format(job.id))
response = 1
if response < 0:
errList.append([response, job.id])
if errList:
badStartFrame = [int(x[1]) for x in errList if x[0] == -1]
badUpdateAttr = [int(x[1]) for x in errList if x[0] == -2]
errStr = "The following errors occurred during the job reset:\n"
if badStartFrame:
errStr += "\tIDs of jobs given start frame was higher than the job's end frame:\n\t\t{}\n".format(badStartFrame)
if badUpdateAttr:
errStr += "\tIDs of jobs where an unknown error occured while trying to udpdate the attributes in the databse:\n\t\t{}".format(badUpdateAttr)
aboutBox(self, "Job Reset Errors", errStr)
#Toggle Archive on Job
elif mode in ["archive", "unarchive"]:
choice = yesNoBox(self, "Confirm",
"Really {0} the selected jobs?".format(mode))
if choice == QMessageBox.No:
return None
archMode = 1 if mode == "archive" else 0
responses = [job.archive(archMode) for job in jobOBJs]
if not all(responses):
failureIDXes = [i for i, x in enumerate(responses) if not x]
failureIDs = [jobIDs[i] for i in failureIDXes]
logger.error("Job Archiver failed on %s", failureIDs)
self.populateJobTree(clear=True)
elif mode == "priority":
for job in jobOBJs:
msgString = "Priority for job {0}:".format(job.id)
reply = intBox(self, "Set Job Priority", msgString, job.priority)
if reply[1]:
job.prioritize(reply[0])
self.populateJobTree()
else:
logger.debug("PrioritizeJob skipped on %s", job.id)
self.doUpdate()