def projectChangeHandler(self, index):
"""Handler for the event where the project selection changed."""
selectedProject = self.projectComboBox.itemText(index)
choice = yesNoBox(self, "Change project", "Reassign this node to " +
selectedProject + "? (will avoid jobs from other"
" projects)")
if choice != QMessageBox.Yes:
aboutBox(self, "No changes", "This node will remain assigned to " +
self.thisNode.project + ".")
self.projectComboBox.setCurrentIndex(self.lastProjectIndex)
return
# get the most up to date info from the database
thisNode = None
try:
thisNode = getThisNodeData()
except sqlerror as err:
logger.debug(str(err))
self.sqlErrorBox()
return
thisNode.project = self.projectComboBox.currentText()
with transaction() as t:
thisNode.update(t)
self.lastProjectIndex = self.projectComboBox.currentIndex()
aboutBox(self, "Success", "Node reassigned to " + selectedProject)
def killJob(job_id):
"""Kills every task associated with job_id. Killed tasks have status code
'K'. If a task was already started, an a kill request is sent to the host
running it.
@return: False if no errors while killing started tasks, else True"""
# mark all of the Ready tasks as Killed
with transaction() as t:
t.cur.execute("""update Hydra_rendertask set status = 'K'
where job_id = '%d' and status = 'R'""" % job_id)
# get hostnames for tasks that were already started
tuples = None # @UnusedVariable
with transaction() as t:
t.cur.execute("""select host from Hydra_rendertask
where job_id = '%d' and status = 'S'""" % job_id)
tuples = t.cur.fetchall()
# make flat list out of single-element tuples fetched from db
hosts = [t for (t,) in tuples]
# send a kill request to each host, note if any failures occurred
error = False
for host in hosts:
try:
error = error or not sendKillQuestion(host)
except socketerror:
logger.debug("There was a problem communicating with {:s}"
.format(host))
error = True
return error
def online(self):
"""Changes the local render node's status to online if it wasn't on-line already"""
with transaction():
[thisNode] = Hydra_rendernode.fetch ("where host = '%s'" % Utils.myHostName( ))
if thisNode.status == OFFLINE:
thisNode.status = IDLE
thisNode.update()
else:
logger.debug("Node is already online.")
self.updateRenderNodeInfo()
def killCurrentJob(self, statusAfterDeath):
"""Kills the render node's current job if it's running one."""
logger.debug("killing %r", self.childProcess)
print "Status after death should be: {0:s}".format(statusAfterDeath)
if self.childProcess:
self.childProcess.kill()
self.childKilled = True
self.statusAfterDeath = statusAfterDeath
else:
logger.debug("no process was running.")
def refreshHandler(self, *args):
try:
jobs = Hydra_job.fetch()
self.jobTable.setRowCount(len(jobs))
for pos, job in enumerate(jobs):
ticket = pickle.loads(job.pickledTicket)
self.jobTable.setItem(pos, 0, QTableWidgetItem_int(str(job.id)))
self.jobTable.setItem(pos, 1, QTableWidgetItem_int(str(job.priority)))
self.jobTable.setItem(pos, 2, QTableWidgetItem(ticket.name()))
except sqlerror as err:
logger.debug(str(err))
aboutBox(self, "SQL error", str(err))
def doFetch( self ):
"""Aggregate method for updating all of the widgets."""
try:
self.updateThisNodeInfo()
self.updateRenderNodeTable()
self.updateRenderTaskGrid()
self.updateJobTable()
self.updateStatusBar()
except sqlerror as err:
logger.debug(str(err))
self.sqlErrorBox()
def withOutputToFile( filename, command ):
"run a command, sending output to a file"
logger.debug( 'writing log file %s', filename )
with file( filename, 'w' ) as f:
f.write( "Launching command %r\n\n" % command )
f.flush( )
# the 'with' statement closes f if there's an error
return subprocess.call( command,
stdout = f,
stderr = subprocess.STDOUT,
)
def updateStatusBar(self):
with transaction() as t:
t.cur.execute ("""select count(status), status
from Hydra_rendernode
group by status""")
counts = t.cur.fetchall ()
logger.debug (counts)
countString = ", ".join (["%d %s" % (count, niceNames[status])
for (count, status) in counts])
time = dt.now().strftime ("%H:%M")
msg = "%s as of %s" % (countString, time)
self.statusLabel.setText (msg)
def getDbInfo():
# open config file
config = ConfigParser.RawConfigParser()
# creat a copy if it doesn't exist
if not os.path.exists (SETTINGS):
folder = os.path.dirname (SETTINGS)
logger.debug ('check for folder %s' % folder)
if os.path.exists (folder):
logger.debug ('exists')
else:
logger.debug ('make %s' % folder)
os.mkdir (folder)
cfgFile = os.path.join (os.path.dirname (sys.argv[0]),
os.path.basename (SETTINGS))
logger.debug ('copy %s' % cfgFile)
shutil.copyfile (cfgFile, SETTINGS)
config.read(SETTINGS)
# get server & db names
host = config.get(section="database", option="host")
db = config.get(section="database", option="db")
return host, db
def killJobButtonHandler(self):
item = self.jobTable.currentItem()
if item and item.isSelected():
row = self.jobTable.currentRow()
id = int(self.jobTable.item(row, 0).text()) # @ReservedAssignment
choice = yesNoBox(self, "Confirm", "Really kill job {:d}?".format(id))
if choice == QMessageBox.Yes:
try:
if killJob(id):
aboutBox(self, "Error", "Some nodes couldn't kill " + "their tasks.")
except sqlerror as err:
logger.debug(str(err))
aboutBox(self, "SQL Error", str(err))
finally:
self.jobCellClickedHandler(item.row(), 0)
def handle( self ):
logger.info ("request")
try:
questionBytes = self.rfile.read( )
question = pickle.loads( questionBytes )
logger.debug(question)
answer = question.computeAnswer( self.TCPserver )
answerBytes = pickle.dumps( answer )
self.wfile.write( answerBytes )
except:
logger.error( """Exception caught:
%s""", traceback.format_exc( ) )
def getOff(self):
"""
Offlines the node and sends a message to the render node server running on localhost to
kill its current task
"""
self.offline()
try:
self.connection = TCPConnection()
killed = self.getAnswer(KillCurrentJobQuestion(statusAfterDeath=READY))
if not killed:
logger.debug("There was a problem killing the task.")
QMessageBox.about(self, "Error", "There was a problem killing the task.")
except socketerror:
QMessageBox.about(self, "Error", "The render node software is not running or has become unresponsive.")
self.updateRenderNodeInfo()
def insert (self, transaction):
names = self.attributes ()
values = [getattr (self, name)
for name in names]
nameString = ", ".join (names)
valueString = ", ".join (len(names) * ["%s"])
query = "insert into %s (%s) values (%s)" % (self.tableName (),
nameString,
valueString)
logger.debug (query)
transaction.cur.executemany (query, [values])
if self.autoColumn:
transaction.cur.execute ("select last_insert_id()")
[id] = transaction.cur.fetchone ()
self.__dict__[self.autoColumn] = id
def getMayaProjectPath(self, scenePath):
"""
Walks up the file tree looking for workspace.mel,
returns the path if found"""
# remove Maya scene file name from the end of the path
mayaProjectPath = os.path.dirname (scenePath)
lastPath = None
wrkspc = "workspace.mel"
while not os.path.exists(os.path.join(mayaProjectPath, wrkspc)):
logger.debug ("%s not in %s", wrkspc, mayaProjectPath)
lastPath = mayaProjectPath
mayaProjectPath = os.path.dirname (mayaProjectPath)
if lastPath == mayaProjectPath:
return ""
return mayaProjectPath
def onlineThisNodeButtonClicked(self):
"""Changes the local render node's status to online if it was offline,
goes back to started if it was pending offline."""
# get most current info from the database
thisNode = None
try:
thisNode = getThisNodeData()
except sqlerror as err:
logger.debug(str(err))
self.sqlErrorBox()
return
if thisNode:
onlineNode(thisNode)
self.doFetch()
def offlineThisNodeButtonClicked(self):
"""Changes the local render node's status to offline if it was idle,
pending if it was working on something."""
# get the most current info from the database
thisNode = None
try:
thisNode = getThisNodeData()
except sqlerror as err:
logger.debug(str(err))
self.sqlErrorBox()
return
if thisNode:
offlineNode(thisNode)
self.doFetch()
def update (self, transaction):
names = list (self.__dirty__)
if not names:
return
values = ([getattr (self, name)
for name in names]
+
[getattr (self, self.primaryKey)])
assignments = ", ".join(["%s = %%s" % name
for name in names])
query = "update %s set %s where %s = %%s" % (self.tableName (),
assignments,
self.primaryKey)
logger.debug ((query, values))
transaction.cur.executemany (query, [values])
def testRenderCommand( self ):
"""Tests a Render Command, a command that invokes the Maya renderer."""
command = [
r'c:\program files\autodesk\maya2011\bin\render.exe',
'-mr:v', '5',
r'\\flex2\ProjectHydra\TestMayaFiles\Chair2.ma'
]
render_task = Hydra_rendertask()
render_task.status = 'R'
render_task.command = repr( command )
render_task.insert()
logger.debug(render_task)
render_question = Questions.RenderQuestion( render_task.id )
render_answer = self.getAnswer( render_question )
logger.debug( render_answer )
def createTasks( self, job ):
starts = range( self.startFrame, self.endFrame + 1, self.batchSize )
ends = [min( start + self.batchSize - 1,
self.endFrame )
for start in starts
]
for start, end in zip( starts, ends ):
command = self.renderCommand(start, end)
logger.debug( command )
task = Hydra_rendertask( status = READY,
command = repr( command ),
job_id = job.id,
priority = self.priority,
project = self.project,
createTime = job.createTime,
requirements = job.requirements,
)
with transaction() as t:
task.insert(transaction=t)
def fetch (cls, whereClause = "", order = None, limit = None,
explicitTransaction=None):
orderClause = "" if order is None else " " + order + " "
limitClause = "" if limit is None else " limit %d " % limit
select = "select * from %s %s %s %s" % (cls.tableName (),
whereClause,
orderClause,
limitClause)
logger.debug (select)
def doFetch (t):
t.cur.execute (select)
names = [desc[0] for desc in t.cur.description]
return [cls (**dict (zip (names, tuple)))
for tuple in t.cur.fetchall ()]
if explicitTransaction:
return doFetch (explicitTransaction)
else:
with transaction() as t:
return doFetch (t)
def getOffRenderNodesButtonClicked(self):
"""For all nodes with boxes checked in the render nodes table, changes
status to offline if idle, or pending if started, and attempts to kill
any task that is running on each node."""
hosts = getCheckedItems(table=self.renderNodeTable, itemColumn=1,
checkBoxColumn=0)
if len(hosts) == 0:
self.noneCheckedBox()
return
choice = yesNoBox(self, "Confirm", "<B>WARNING</B>: All progress on"
" current tasks will be lost for the selected"
" render nodes. Are you sure you want to stop these"
" nodes? <br>" + str(hosts))
if choice != QMessageBox.Yes:
aboutBox(self, "Aborted", "No action taken.")
return
error = False
notKilledList = list()
with transaction() as t:
rendernode_rows = Hydra_rendernode.fetch(explicitTransaction=t)
for node_row in rendernode_rows:
if node_row.host in hosts:
offlineNode(node_row)
try:
killed = sendKillQuestion(node_row.host, READY)
error = error or not killed
except socketerror as err:
logger.debug(str(err) + '\n'
+ "Error while trying to contact "
+ node_row.host)
notKilledList.append(node_row.host)
error = True
if error:
aboutBox(self, "Error", "The following nodes could not be stopped"
" for some reason. Look in FarmView.log for more details."
"<br>" + str(notKilledList))
self.doFetch()
def killTaskButtonHandler(self):
item = self.taskTable.currentItem()
if item and item.isSelected():
row = self.taskTable.currentRow()
id = int(self.taskTable.item(row, 0).text()) # @ReservedAssignment
choice = yesNoBox(self, "Confirm", "Really kill task {:d}?".format(id))
if choice == QMessageBox.Yes:
try:
killTask(id)
except socketerror as err:
logger.debug(str(err))
aboutBox(
self,
"Error",
"Task couldn't be killed because "
"there was a problem communicating with the host running "
"it.",
)
except sqlerror as err:
logger.debug(str(err))
aboutBox(self, "SQL Error", str(err))
def resurrectTaskButtonHandler(self):
taskItem = self.taskTable.currentItem()
if taskItem and taskItem.isSelected():
row = self.taskTable.currentRow()
id = int(self.taskTable.item(row, 0).text()) # @ReservedAssignment
choice = yesNoBox(self, "Confirm", "Resurrect task {:d}?".format(id))
if choice == QMessageBox.Yes:
error = None
try:
error = resurrectTask(id)
except sqlerror as err:
logger.debug(str(err))
aboutBox(self, "SQL Error", str(err))
finally:
if error:
msg = "Task couldn't be resurrected because it's " "either not dead or is currently running."
logger.debug(msg)
aboutBox(self, "Error", msg)
else:
jobItem = self.jobTable.currentItem()
self.jobCellClickedHandler(jobItem.row(), 0)
def killTaskButtonHandler (self):
item = self.taskTable.currentItem ()
if item and item.isSelected ():
row = self.taskTable.currentRow ()
task_id = int(self.taskTable.item(row, 0).text())
choice = yesNoBox(self, "Confirm", "Really kill task {:d}?"
.format(task_id))
if choice == QMessageBox.Yes:
killed = None
try:
killed = killTask(task_id)
if not killed:
# TODO: make a better error message
aboutBox(self, "Error", "Task couldn't be killed for "
"some reason.")
except socketerror as err:
logger.debug(str(err))
aboutBox(self, "Error", "Task couldn't be killed because "
"there was a problem communicating with the host running "
"it.")
except sqlerror as err:
logger.debug(str(err))
aboutBox(self, "SQL Error", str(err))
def __exit__ (self, errorType, value, traceback):
if errorType is None:
logger.debug ("commit %s", self)
self.cur.execute ("commit")
else:
logger.debug ("rollback %s", self)
self.cur.execute ("rollback")
logger.debug ("exit transaction %s", self)
self.db.close()
def updateThisNodeInfo(self):
"""Updates widgets on the "This Node" tab with the most recent
information available."""
# if the buttons are disabled, don't bother
if not self.thisNodeButtonsEnabled:
return
# get the most current info from the database
thisNode = None
try:
thisNode = getThisNodeData()
self.updateProjectComboBox()
except sqlerror as err:
logger.debug(str(err))
self.sqlErrorBox()
if thisNode:
# update the labels
self.nodeNameLabel.setText(thisNode.host)
self.nodeStatusLabel.setText(niceNames[thisNode.status])
self.updateTaskIDLabel(thisNode.task_id)
self.nodeVersionLabel.setText(
getSoftwareVersionText(thisNode.software_version))
self.setCurrentProjectSelection(thisNode.project)
self.updateRestrictToProjectLabel(thisNode.restrict_to_project)
self.updateMinPriorityLabel(thisNode.minPriority)
self.updateCapabilitiesLabel(thisNode.capabilities)
else:
QMessageBox.about(self, "Notice",
"Information about this node cannot be displayed because it is"
"not registered on the render farm. You may continue to use"
" Farm View, but it must be restarted after this node is "
"registered if you wish to see this node's information.")
self.setThisNodeButtonsEnabled(False)
def sendKillQuestion(renderhost, newStatus=KILLED):
"""Tries to kill the current task running on the renderhost. Returns True
if successful, otherwise False"""
logger.debug ('kill job on %s' % renderhost)
connection = TCPConnection(hostname=renderhost)
answer = connection.getAnswer(
KillCurrentJobQuestion(newStatus))
logger.debug("child killed: %s" % answer.childKilled)
if not answer.childKilled:
logger.debug("%r tried to kill its job but failed for some reason."
% renderhost)
return answer.childKilled
def doSubmit( self ):
"""Submits a job ticket for this scene to be split into
tasks and processed."""
logger.debug ('doSubmit')
reqs = sorted ([str (req.text ())
for req in self.requirementsListWidget.selectedItems ()])
reqs_pattern = '%' + '%'.join (reqs) + '%' if reqs else '%'
print reqs_pattern
sceneFile = str( self.sceneText.text() ).replace ('\\', '/')
startFrame = self.startSpinBox.value( )
endFrame = self.endSpinBox.value( )
numJobs = self.numJobsSpinBox.value( )
batchSize = int(math.ceil((endFrame - startFrame + 1.0) / numJobs))
logger.debug ("numJobs %s batchSize %s", numJobs, batchSize)
priority = self.prioritySpinBox.value( )
project = str(self.projectComboBox.currentText())
executable = str(self.executableComboBox.currentText())
mayaProjectPath = str(self.projectDirLineEdit.text())
if not os.path.exists(os.path.join (mayaProjectPath, "workspace.mel")):
# try to find workspace.mel
mayaProjectPath = self.getMayaProjectPath(sceneFile)
if not mayaProjectPath:
logger.debug("workspace.mel not found")
aboutBox(self, "Error", """
The project path cannot be set because workspace.mel could not
be located. Please set the project path manually.""")
return
if yesNoBox(self, "Confirm",
"Maya project path set to:<br>" +
mayaProjectPath +
"<br> Is this correct?") == QMessageBox.No:
aboutBox(self, "Abort",
"Submission aborted. Please set the Maya project path manually.")
return
self.projectDirLineEdit.setText(mayaProjectPath)
# executable names a class in the JobTicket module
ticketClass = getattr (JobTicket, executable)
ticketClass(sceneFile, mayaProjectPath, startFrame, endFrame, batchSize,
priority, project, reqs_pattern).submit()
aboutBox(self, "Success",
"Job submitted. Please close the submitter window.")
def getOffThisNodeButtonClicked(self):
"""Offlines the node and sends a message to the render node server
running on localhost to kill its current task (task will be
resubmitted)"""
thisNode = None
try:
thisNode = getThisNodeData()
except sqlerror as err:
logger.debug(str(err))
self.sqlErrorBox()
return
choice = yesNoBox(self, "Confirm", "All progress on the current job"
" will be lost. Are you sure you want to stop it?")
if choice != QMessageBox.Yes:
aboutBox(self, "Abort", "No action taken.")
return
if thisNode:
offlineNode(thisNode)
if thisNode.task_id:
try:
# TODO: use JobKill for getOff instead of doing it manually
killed = sendKillQuestion(renderhost = "localhost",
newStatus = READY)
if not killed:
logger.debug("There was a problem killing the task.")
aboutBox(self, "Error", "There was a problem killing"
" the task.")
else:
aboutBox(self, "Success", "Job was successfully"
" stopped. Node offlined.")
except socketerror:
logger.debug(socketerror.message)
aboutBox(self, "Error", "There was a problem communicating"
" with the render node software. Either it's not"
" running, or it has become unresponsive.")
else:
aboutBox(self, "Offline", "No job was running. Node offlined.")
self.doFetch()
if not startDir:
startDir = os.getcwd()
else:
startDir = currentDir
mayaProjectPath = QFileDialog.getOpenFileName(
parent=self,
caption="Find workspace.mel",
directory=startDir,
filter="workspace.mel")
if mayaProjectPath:
# remove "workspace.mel" from the end of the path
mayaProjectPath = str(mayaProjectPath).split('/')
mayaProjectPath.pop()
mayaProjectPath = '/'.join(mayaProjectPath) + '/'
self.projectDirLineEdit.setText(mayaProjectPath)
if __name__ == '__main__':
try:
logger.debug(sys.argv) # prints out argv
app = QApplication( sys.argv )
window = SubmitterWindow( )
window.show( )
retcode = app.exec_( )
sys.exit( retcode )
except Exception, e:
logger.error( traceback.format_exc( e ) )
raise
