def run(self):
'''
The run function constantly grabs new jobs from the jobQueue until it receives the directory path "terminate", at which point it
shuts down. Jobs are passed as the path to the input files ticketNumber.data and ticketNumber.args (everything up to the '.' is
passed).
'''
while 1:
dirpath = self.server.jobQueue.get(True) #block this thread's execution until it gets something from the queue
#if dirpath is just 'terminate', then the server is telling the thread to shut down gracefully
if dirpath == 'terminate':
self.log('Thread ' + str(self.threadId) + ' exiting gracefully...')
self.server.jobQueue.task_done()
self.clear()
break
# parse out the ticket number
(junk, ticketNumber) = (dirpath.rstrip('/')).rsplit('/', 1) #the ticket number is what's at the end of the path to the input file
#each element in jobDistribution is a tuple of ( ticketNumber, time recorded, extensionsGranted)
self.server.watchdog.updateThread(self.threadId, ticket=ticketNumber, extensionsGranted=0)
#jobs are passed as dirpath, the path to the ticket's input files e.g. "/doos/workspace/files/input/23424"
#read the arguments from the .args file for this ticket
file = open(dirpath + ticketNumber + '.args', 'r')
args = file.read()
file.close()
#parse the arguments
args = utils.makeDictionary(args)
#now write the time that the file was taken out of the queue
file = open(self.home + 'files/output/' + str(ticketNumber) + '/status.txt', 'a')
file.write('timeDequeued:' + datetime.now().isoformat() + "\n")
file.close()
#execution stage
success = self.scriptRunner.execute(dirpath, args)
#write the job's status to status.txt now that it's done
file = open(self.home + 'files/output/' + str(ticketNumber) + '/status.txt', 'a')
if success:
file.write('timeCompleted:')
self.log('Thread ' + self.threadId + ' successfully processed job ' + ticketNumber)
else:
file.write('timeFailed:')
self.log('Thread ' + self.threadId + ' has abandoned job ' + ticketNumber, 'error\t')
file.write( datetime.now().isoformat() + "\n" )
file.close()
#remove the input directory now that we're done with them
shutil.rmtree(dirpath)
self.server.jobQueue.task_done() #helps the queue keep track of how many jobs are still running
self.server.watchdog.updateThread(self.threadId, ticket='ready', extensionsGranted=0)
def run(self):
try:
self.log("Client connected from " + self.remoteHostName)
data_chunks = [] #we're just concatenating the chunks of data received, but this is faster than string concatenation
buf = self.client.recv(self.socketBufferSize)
bytesExpected = None
bytesGotten = 0
while True: #the first part of the message, up to '|', is the length of the transmission. Keep going till it's been exceeded
if buf == '':
raise socket.error("Socket has been broken before transmission completed. It probably failed.")
data_chunks.append(buf)
#find out how many bytes will be sent
if bytesExpected is None: #if we still haven't determined the number of bytes to expect
buf = "".join(data_chunks) #in case the bytesExpected number was in another buffering, join whatever we've got
i = buf.find("|") #fields are delimited by | like so: bytesExpected|checkSum|argString|data
#args is structred thus: "key1=value1;key2=value2;key3;key4;key5=value5" etc
if i is not -1: #if | is found, then we have the bytesExpected
bytesExpected = int(buf[:i]) #bytesExpected is the string from the beginning until the |
data_chunks = [ buf[i+1:] ] #we probably have more than just bytesExpected in the buffer, so store that data
bytesGotten = len( data_chunks[0] )
else:
bytesGotten += len(buf)
self.log('Received ' + str(bytesGotten) + ' bytes of expected ' + str(bytesExpected) + ' bytes from client ' + self.remoteHostName + '.')
#if we're still determining the number of bytes to expect or we're still downloading the file,
if bytesExpected is None or bytesGotten < bytesExpected:
buf = self.client.recv(self.socketBufferSize) #get more data
else:
break #otherwise we're done.
#data_chunks is a list of chunks of the data. Join them together.
data = "".join(data_chunks)
checksum, data = data.split('|', 1) #pull out the checksum from the data
#get the sha1 hash for checksumming
m = hashlib.sha1()
m.update(data)
if (m.hexdigest() != checksum):
try:
self.client.sendall(formatResponse("[ERROR] Checksum failed."))
finally:
raise Exception("Checksum failed! ")
argString, data = data.split('::file start::', 1)
#convert argString into a key-value dictionary
args = utils.makeDictionary(argString)
#guard against "terminate=false"
if args.has_key('terminate') and args['terminate']:
try:
self.client.sendall(formatResponse("[OK]"))
finally:
self.log("Received shutdown command from '" + self.remoteHostName + "'.")
self.server.terminate() #tell the server to shut down all threads
#if they want to put it in the queue, dump the data to a file and enqueue
elif args.has_key('prepareJob') and args['prepareJob'] is True:
self.prepareJob(argString, data)
#if they want status or they want the job
elif ( (args.has_key('statusJob') and args['statusJob']) or (args.has_key('returnJob') and args['returnJob']) ) and args.has_key('ticket'):
logfile, status = self.statusJob(args['ticket']) #get the file containing the job's status
if status == jobStatus.notFound:
self.client.sendall(formatResponse("[ERROR] Unknown Ticket: " + args['ticket'] + '.'))
self.log("Sent 'ticket not found' for ticket '" + args['ticket'] + "' to '" + self.remoteHostName + "'.", 'error\t')
elif (args.has_key('returnJob') and args['returnJob']) and status == jobStatus.done:
#this function handles its own logging.
self.returnJob(args['ticket'])
else:
self.client.sendall(formatResponse("[OK] " + str(jobStatus.asText(status))))
self.log("Sent status for ticket '" + args['ticket'] + "' to '" + self.remoteHostName + "'.")
# command to delete output files of job from files/output
# TODO: have command remove job from queue if job not processed yet
elif (args.has_key('deleteJob') and args['deleteJob']) and args.has_key('ticket'):
#this function handles its own logging.
self.deleteJob(args['ticket'])
else:
self.client.sendall(formatResponse("[ERROR] Unknown command."))
except socket.error, (message):
self.log("Socket error for host '" + self.remoteHostName + "': " + str(message), "error\t")
