示例#1
0
	def run(self):
		'''
		The run function constantly grabs new jobs from the jobQueue until it receives the directory path "terminate", at which point it 
		shuts down.  Jobs are passed as the path to the input files ticketNumber.data and ticketNumber.args (everything up to the '.' is
		passed).
		'''
		while 1:
			dirpath = self.server.jobQueue.get(True) #block this thread's execution until it gets something from the queue

			#if dirpath is just 'terminate', then the server is telling the thread to shut down gracefully
			if dirpath == 'terminate':
				self.log('Thread ' + str(self.threadId) + ' exiting gracefully...')
				self.server.jobQueue.task_done()
				self.clear()
				break

			# parse out the ticket number
			(junk, ticketNumber) = (dirpath.rstrip('/')).rsplit('/', 1) #the ticket number is what's at the end of the path to the input file

			#each element in jobDistribution is a tuple of ( ticketNumber, time recorded, extensionsGranted)
			self.server.watchdog.updateThread(self.threadId, ticket=ticketNumber, extensionsGranted=0)

			#jobs are passed as dirpath, the path to the ticket's input files e.g. "/doos/workspace/files/input/23424"
			#read the arguments from the .args file for this ticket
			file = open(dirpath + ticketNumber + '.args', 'r')
			args = file.read()
			file.close()

			#parse the arguments
			args = utils.makeDictionary(args)

			#now write the time that the file was taken out of the queue
			file = open(self.home + 'files/output/' + str(ticketNumber) + '/status.txt', 'a')
			file.write('timeDequeued:' + datetime.now().isoformat() + "\n")
			file.close()

			#execution stage
			success = self.scriptRunner.execute(dirpath, args)

			#write the job's status to status.txt now that it's done
			file = open(self.home + 'files/output/' + str(ticketNumber) + '/status.txt', 'a')

			if success:
				file.write('timeCompleted:')
				self.log('Thread ' + self.threadId + ' successfully processed job ' + ticketNumber)
			else:
				file.write('timeFailed:')
				self.log('Thread ' + self.threadId + ' has abandoned job ' + ticketNumber, 'error\t')

			file.write( datetime.now().isoformat() + "\n" )
			file.close()

			#remove the input directory now that we're done with them
			shutil.rmtree(dirpath)

			self.server.jobQueue.task_done() #helps the queue keep track of how many jobs are still running
			self.server.watchdog.updateThread(self.threadId, ticket='ready', extensionsGranted=0)
示例#2
0
	def run(self):
		try:
			self.log("Client connected from " + self.remoteHostName)
			data_chunks = [] #we're just concatenating the chunks of data received, but this is faster than string concatenation
			buf = self.client.recv(self.socketBufferSize)
			bytesExpected = None
			bytesGotten = 0

			while True: #the first part of the message, up to '|', is the length of the transmission.  Keep going till it's been exceeded
				if buf == '':
					raise socket.error("Socket has been broken before transmission completed. It probably failed.")
				data_chunks.append(buf)

				#find out how many bytes will be sent
				if bytesExpected is None:  #if we still haven't determined the number of bytes to expect
					buf = "".join(data_chunks) #in case the bytesExpected number was in another buffering, join whatever we've got
					i = buf.find("|")  #fields are delimited by | like so: bytesExpected|checkSum|argString|data
															#args is structred thus: "key1=value1;key2=value2;key3;key4;key5=value5"  etc
					if i is not -1:  #if | is found, then we have the bytesExpected
						bytesExpected = int(buf[:i])  #bytesExpected is the string from the beginning until the |
						data_chunks = [ buf[i+1:] ]  #we probably have more than just bytesExpected in the buffer, so store that data
						bytesGotten = len( data_chunks[0] )
				else:
					bytesGotten += len(buf)

				self.log('Received ' + str(bytesGotten) + ' bytes of expected ' + str(bytesExpected) + ' bytes from client ' + self.remoteHostName + '.')

				#if we're still determining the number of bytes to expect or we're still downloading the file,
				if bytesExpected is None or bytesGotten < bytesExpected:
					buf = self.client.recv(self.socketBufferSize)  #get more data
				else:
					break  #otherwise we're done.

			#data_chunks is a list of chunks of the data.  Join them together.
			data = "".join(data_chunks)
			checksum, data = data.split('|', 1) #pull out the checksum from the data

			#get the sha1 hash for checksumming
			m = hashlib.sha1()
			m.update(data)
			if (m.hexdigest() != checksum):
				try:
					self.client.sendall(formatResponse("[ERROR] Checksum failed."))
				finally:
					raise Exception("Checksum failed! ")

			argString, data = data.split('::file start::', 1)

			#convert argString into a key-value dictionary
			args = utils.makeDictionary(argString)

			#guard against "terminate=false"
			if args.has_key('terminate') and args['terminate']:
				try:
					self.client.sendall(formatResponse("[OK]"))
				finally:
					self.log("Received shutdown command from '" + self.remoteHostName + "'.")
					self.server.terminate() #tell the server to shut down all threads

			#if they want to put it in the queue, dump the data to a file and enqueue
			elif args.has_key('prepareJob') and args['prepareJob'] is True:
				self.prepareJob(argString, data)

			#if they want status or they want the job
			elif ( (args.has_key('statusJob') and args['statusJob']) or (args.has_key('returnJob') and args['returnJob']) ) and args.has_key('ticket'):
				logfile, status = self.statusJob(args['ticket']) #get the file containing the job's status

				if status == jobStatus.notFound:
					self.client.sendall(formatResponse("[ERROR] Unknown Ticket: " + args['ticket'] + '.'))
					self.log("Sent 'ticket not found' for ticket '" + args['ticket'] + "' to '" + self.remoteHostName + "'.", 'error\t')
				elif (args.has_key('returnJob') and args['returnJob']) and status == jobStatus.done:
					#this function handles its own logging.
					self.returnJob(args['ticket'])
				else:
					self.client.sendall(formatResponse("[OK] " + str(jobStatus.asText(status))))
					self.log("Sent status for ticket '" + args['ticket'] + "' to '" + self.remoteHostName + "'.")

			# command to delete output files of job from files/output
			# TODO: have command remove job from queue if job not processed yet
			elif (args.has_key('deleteJob') and args['deleteJob']) and args.has_key('ticket'):
				#this function handles its own logging.
				self.deleteJob(args['ticket'])
			else:
				self.client.sendall(formatResponse("[ERROR] Unknown command."))

		except socket.error, (message):
			self.log("Socket error for host '" + self.remoteHostName + "': " + str(message), "error\t")