def getAnswer( self, question ):
# send the question to a remote server and get an answer back
# create a TCP socket
sock = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
try:
# connect to the server
logger.info( 'connect to %s %s', self.hostname, self.port )
sock.connect( ( self.hostname, self.port ) )
# convert the question to ASCII
questionBytes = pickle.dumps( question )
# send the question
sock.sendall( questionBytes )
# close the sending half of the connection so the other side
# knows we're done sending
sock.shutdown( socket.SHUT_WR )
# read the response, an ASCII encoded object
answerBytes = sock.recv( Constants.MANYBYTES )
# convert the response to an object
answer = pickle.loads( answerBytes )
finally:
sock.close( )
return answer
def main ():
logger.info ('starting in %s', os.getcwd())
logger.info ('arglist %s', sys.argv)
socketServer = RenderTCPServer( )
# socketServer.serverThread.join( )
socketServer.createIdleLoop (5, socketServer.processRenderTasks )
pulseThread = threading.Thread(target = heartbeat, name = "heartbeat",
args = (60,))
pulseThread.start()
def __init__( self,
port = Constants.PORT,
):
MyTCPHandler.TCPserver = self
logger.info( 'open socket %r %s', "", port )
self.serverObject = MySocketServer( ( "", port),
MyTCPHandler)
self.serverThread = threading.Thread( target = runTheServer,
name = "server thread",
args = ( self.serverObject, )
)
self.serverThread.start( )
def handle( self ):
logger.info ("request")
try:
questionBytes = self.rfile.read( )
question = pickle.loads( questionBytes )
logger.debug(question)
answer = question.computeAnswer( self.TCPserver )
answerBytes = pickle.dumps( answer )
self.wfile.write( answerBytes )
except:
logger.error( """Exception caught:
%s""", traceback.format_exc( ) )
def __init__(self, *arglist, **kwargs):
# check for another instance of RenderNodeMain.exe
nInstances = len (filter (lambda line: 'RenderNodeMain' in line,
subprocess.check_output ('tasklist').split('\n')))
logger.info ("%d RenderNodeMain instances running." % nInstances)
if nInstances > 1:
logger.info("Blocked RenderNodeMain from running because another"
" instance already exists.")
sys.exit(1)
if nInstances == 0 and not sys.argv[0].endswith('.py'):
logger.error("Can't find running RenderNodeMain.")
sys.exit(1)
TCPServer.__init__(self, *arglist, **kwargs)
self.childProcess = None
self.childKilled = False
self.statusAfterDeath = None # must be a status from MySQLSetup
# clean up in case we had an unexpected termination last time around
[thisNode] = Hydra_rendernode.fetch ("where host = '%s'"
% Utils.myHostName())
if thisNode.task_id:
if thisNode.status == PENDING or thisNode.status == OFFLINE:
newStatus = OFFLINE
else:
newStatus = IDLE
unstick (taskID=thisNode.task_id, newTaskStatus=CRASHED,
host=thisNode.host, newHostStatus=newStatus)
# update current software version if necessary
current_version = sys.argv[0]
if thisNode.software_version != current_version:
thisNode.software_version = current_version
with transaction() as t:
thisNode.update(t)
def processRenderTasks(self):
[thisNode] = Hydra_rendernode.fetch ("where host = '%s'"
% Utils.myHostName( ))
logger.info("""Host: %r
Status: %r
Project: %r
Capabilities %r""",
thisNode.host, thisNode.status, thisNode.project,
thisNode.capabilities)
# If this node is not idle, don't try to find a new job
if thisNode.status != IDLE:
return
# otherwise, get a job that's:
## ready to be run and
## has a high enough priority level for this particular node and
## (optionally) is on this node's assigned project
queryString = ("where status = '%s' and priority >= %s"
% (READY, thisNode.minPriority))
queryString += " and '%s' like requirements" % thisNode.capabilities
if thisNode.restrict_to_project:
queryString += " and project = '%s'" % thisNode.project
orderString = ("order by project = '%s' desc, priority desc, length(requirements) desc, id asc" %
thisNode.project)
with transaction() as t:
render_tasks = Hydra_rendertask.fetch (
queryString,
limit=1,
order=orderString,
explicitTransaction=t)
if not render_tasks:
return
render_task = render_tasks[0]
# create log for this task and update task entry in the database
if not os.path.isdir( RENDERLOGDIR ):
os.makedirs( RENDERLOGDIR )
render_task.logFile = os.path.join(RENDERLOGDIR, '%010d.log.txt'
% render_task.id )
render_task.status = STARTED
render_task.host = thisNode.host
thisNode.status = STARTED
thisNode.task_id = render_task.id
render_task.startTime = datetime.datetime.now()
render_task.update(t)
thisNode.update(t)
logger.debug ('working on render task %s', render_task.id)
log = file(render_task.logFile, 'w')
try:
log.write('Hydra log file %s on %s\n'
% ( render_task.logFile, render_task.host ) )
log.write('RenderNodeMain is %s\n' % sys.argv)
log.write ("Initial drive mappings (net use):\n\n")
flushOut(log)
subprocess.call("net use", stdout = log,
stderr = subprocess.STDOUT)
if mapDrive ('w:', r'\\oscar.cpc.local\px3'):
log.write("Found w:, assuming it's correct.\n")
else:
log.write("Attempted to map w:\n")
flushOut(log)
subprocess.call("net use", stdout = log,
stderr = subprocess.STDOUT)
flushOut(log)
log.write('Command: %s\n\n' % ( render_task.command ) )
flushOut(log)
# run the job and keep track of the process
self.childProcess = subprocess.Popen( eval( render_task.command ),
stdout = log,
stderr = subprocess.STDOUT )
logger.debug('started PID %s to do task %s',
self.childProcess.pid, render_task.id)
# wait until the job is finished or terminated
render_task.exitCode = self.childProcess.wait()
log.write('\nProcess exited with code %d\n' % render_task.exitCode)
return RenderAnswer( )
except Exception, e:
traceback.print_exc( e, log )
raise
def runTheServer( serverObject ):
logger.info ("off to the races")
serverObject.serve_forever( )