def _fill_work_queue(self): trace.cc(1, "Starting to fill task queue") for pathToMeasure in self._pathsToMeasure: if self._check_command(): self._folderWalker.walk(pathToMeasure) if self._check_command() and self._workPackage.size_items() > 0: self._send_current_package()
def _run(self): # We keep processing queue until the job signals it is done and # the queue is empty, or we receive an abort command while self._continue_processing(): try: if self._workDone and self._outQueue.empty(): break filesOutput = self._outQueue.get_nowait() except Empty: trace.cc(3, "EMPTY OUTPUT") time.sleep(OUTPUT_EMPTY_WAIT) else: self.taskPackagesReceived += 1 trace.cc(2, "GOT {0} measures".format(len(filesOutput))) # We get a set of output for multiple files with each # outputQueue item. Each file has a set of output # and potential errors that we pack to the app for filePath, outputList, errorList in filesOutput: # Synchronus callback to applicaiton # Output writing and screen update occurs in this call self._file_measure_callback(filePath, outputList, errorList) if errorList: trace.file(1, "ERROR measuring: {0}".format(filePath)) self._controlQueue.put_nowait(('JOB', 'ERROR', filePath))
def _wait_process_packages(self): trace.cc(1, "Task queue is complete, processing packages") while self._check_command() and self._task_queue_size() > 0: time.sleep(MAIN_PROCESSING_SLEEP) self._status_callback() if self._check_command(): self._send_workers_command('WORK_DONE') else: self._send_workers_command('EXIT')
def file_measured_callback(self, filePath, measures, analysisResults): ''' Callback from the masurement module We store up a list of tuples with the work output for a given file ''' assert filePath == self._currentFilePath, "Measure callback out of sync" trace.cc(3, "_file_measured_callback: {0}".format(filePath)) trace.file(3, " measures: {0}".format(measures)) trace.file(3, " analysis: {0}".format(analysisResults)) self._currentFileOutput.append((measures, analysisResults))
def file_measured_callback(self, filePath, measures, analysisResults): ''' Callback from the masurement module We store up a list of tuples with the work output for a given file ''' assert filePath == self._currentFilePath, "Measure callback out of sync" trace.cc(3, "_file_measured_callback: {0}".format(filePath)) trace.file(3, " measures: {0}".format(measures)) trace.file(3, " analysis: {0}".format(analysisResults)) self._currentFileOutput.append((measures, analysisResults))
def _file_complete(self): ''' Cache the output from the measurement callbacks for current file ''' if self._currentFileOutput or self._currentFileErrors: self._currentOutput.append( (self._currentFilePath, self._currentFileOutput, self._currentFileErrors)) trace.cc(3, "Caching results: {0}".format(self._currentFilePath)) else: trace.cc(3, "No measures for: {0}".format(self._currentFilePath)) self._currentFileOutput = [] self._currentFileErrors = []
def _file_complete(self): ''' Cache the output from the measurement callbacks for current file ''' if self._currentFileOutput or self._currentFileErrors: self._currentOutput.append( (self._currentFilePath, self._currentFileOutput, self._currentFileErrors)) trace.cc(3, "Caching results: {0}".format(self._currentFilePath)) else: trace.cc(3, "No measures for: {0}".format(self._currentFilePath)) self._currentFileOutput = [] self._currentFileErrors = []
def _post_results(self): ''' Send any cached results back to main process's out thread This is a set of results for config measure of every file in the last work package ''' try: self._outputQueue.put(self._currentOutput, True, OUT_PUT_TIMEOUT) trace.cc(3, "OUT - PUT {0} items".format(len(self._currentOutput))) except Full: raise utils.JobException("FATAL EXCEPTION - Out Queue full, can't put") finally: self._currentOutput = []
def _continue_processing(self): continueProcessing = True otherCommands = [] myCommand = None try: while True: (target, command, payload) = self._controlQueue.get_nowait() trace.cc(3, "command - {0}, {1}".format(target, command)) if target == self.name: myCommand = command break else: otherCommands.append((target, command, payload)) except Empty: pass finally: if 'EXIT' == myCommand: trace.cc(1, "COMMAND: EXIT") continueProcessing = False elif 'WORK_DONE' == myCommand: trace.cc(1, "COMMAND: WORK_DONE") self._workDone = True for (target, command, payload) in otherCommands: trace.cc(3, "putting {0}, {1}".format(target, command)) try: self._controlQueue.put((target, command, payload), True, CONTROL_QUEUE_TIMEOUT) except Full: raise utils.JobException("FATAL EXCEPTION - Control Queue full, can't put") return continueProcessing
def _post_results(self): ''' Send any cached results back to main process's out thread This is a set of results for config measure of every file in the last work package ''' try: self._outputQueue.put(self._currentOutput, True, OUT_PUT_TIMEOUT) trace.cc(3, "OUT - PUT {0} items".format(len(self._currentOutput))) except Full: raise utils.JobException( "FATAL EXCEPTION - Out Queue full, can't put") finally: self._currentOutput = []
def _send_current_package(self): ''' Place package of work on queue, and start a worker ''' self._workers.start_next() trace.cc(2, "PUT WorkPackage - files: {0}, bytes: {1}...".format( self._workPackage.size_items(), self._workPackage.size_bytes())) trace.cc(4, self._workPackage.items()) try: self._taskQueue.put(self._workPackage.items(), True, TASK_FULL_TIMEOUT) except Full: raise utils.JobException("FATAL ERROR -- FULL TASK QUEUE") else: self._taskPackagesSent += 1 self._filesSinceLastSend = 0 self._workPackage.reset()
def __init__(self, inputQueue, outputQueue, controlQueue, context, num, jobName=WORKER_PROC_BASENAME): ''' Init is called in the parent process ''' multiprocessing.Process.__init__(self, name=jobName + str(num)) self._inputQueue = inputQueue self._outputQueue = outputQueue self._controlQueue = controlQueue self._continueProcessing = True self._currentOutput = [] self._currentFilePath = None self._currentFileIterator = None self._currentFileOutput = [] self._currentFileErrors = [] self._dbgContext, self._profileName = context trace.cc(2, "Initialized new process: {0}".format(self.name))
def _put_files_in_queue(self, path, deltaPath, filesAndConfigs): ''' Package files from the given folder into workItems that are then grouped into workPackages that are placed into the task queue for jobworkers. Packages are broken up if files number or total size exceeds thresholds to help evenly distribute load across cores ''' if not filesAndConfigs: return self.numFoldersMeasured += 1 for fileName, configEntrys in filesAndConfigs: # Expensive to check file size here, but it is worth it for # pracelling widely varying file sizes out to cores for CPU intensive # jobs. Profiling shows it is not worth caching this try: fileSize = utils.get_file_size(os.path.join(path, fileName)) except Exception, e: # It is possible (at least in Windows) for a fileName to exist # in the file system but be invalid for Windows calls. This is # the first place we try to access the file through the file # system; if it blows up we don't want the job to fall apart, # and this is an unusual case, so unlike more likely errors, # we don't bother with a pathway back to the main application # to handle the error; we just swallow it and provide debug trace.msg(1, str(e)) continue trace.cc(3, "WorkItem: {0}, {1}".format(fileSize, fileName)) self.numFilesToProcess += 1 workItem = (path, deltaPath, fileName, configEntrys, self._options, len(filesAndConfigs)) self._workPackage.add(workItem, fileSize) if self._workPackage.ready_to_send() or ( self._filesSinceLastSend > MAX_FILES_BEFORE_SEND): self._send_current_package() if not self._check_command(): break
def __init__(self, outQueue, controlQueue, profileName, file_measure_callback): trace.cc(1, "Creating output queue thread") threading.Thread.__init__(self, name="Out") self._profileName = profileName # The main thread manages our life; if something gets truly out of # sync we care more about exiting than ensuring output is flushed self.daemon = True # The main thread owns our queues self._outQueue = outQueue self._controlQueue = controlQueue self._file_measure_callback = file_measure_callback # Total task output packages we've received from all processes self.taskPackagesReceived = 0 # Flag to track when we receive WORK_DONE from the Job self._workDone = False
def _run(self): ''' Process items from input queue until it's empty and app signals all done ''' trace.cc(1, "STARTING: Begining to process input queue...") while self._continueProcessing: try: workPackage = self._inputQueue.get_nowait() trace.cc( 2, "GOT WorkPackage - files: {0}".format(len(workPackage))) except Empty: # The input queue can return empty when it really isn't, or # we are in mid job and have burned down the empty queue # Sleeping after these helps performance with many cores, vs # just blocking on inputQueue.get trace.cc(3, "EMPTY INPUT") time.sleep(INPUT_EMPTY_WAIT) self._check_for_stop() else: for workItem in workPackage: if not self._measure_file(workItem): self._continueProcessing = False break self._post_results() self._check_for_stop()
def _run(self): ''' Process items from input queue until it's empty and app signals all done ''' trace.cc(1, "STARTING: Begining to process input queue...") while self._continueProcessing: try: workPackage = self._inputQueue.get_nowait() trace.cc(2, "GOT WorkPackage - files: {0}".format(len(workPackage))) except Empty: # The input queue can return empty when it really isn't, or # we are in mid job and have burned down the empty queue # Sleeping after these helps performance with many cores, vs # just blocking on inputQueue.get trace.cc(3, "EMPTY INPUT") time.sleep(INPUT_EMPTY_WAIT) self._check_for_stop() else: for workItem in workPackage: if not self._measure_file(workItem): self._continueProcessing = False break self._post_results() self._check_for_stop()
def __init__(self, inputQueue, outputQueue, controlQueue, context, num, jobName=WORKER_PROC_BASENAME): ''' Init is called in the parent process ''' multiprocessing.Process.__init__(self, name=jobName + str(num)) self._inputQueue = inputQueue self._outputQueue = outputQueue self._controlQueue = controlQueue self._continueProcessing = True self._currentOutput = [] self._currentFilePath = None self._currentFileIterator = None self._currentFileOutput = [] self._currentFileErrors = [] self._dbgContext, self._profileName = context trace.cc(2, "Initialized new process: {0}".format(self.name))
class Worker(Process): ''' The worker class executes as separate processes spawned by the Job They take items from the input queue, delegate calls to the measurement modules, and package measures for the output queue. ''' def __init__(self, inputQueue, outputQueue, controlQueue, context, num, jobName=WORKER_PROC_BASENAME): ''' Init is called in the parent process ''' multiprocessing.Process.__init__(self, name=jobName + str(num)) self._inputQueue = inputQueue self._outputQueue = outputQueue self._controlQueue = controlQueue self._continueProcessing = True self._currentOutput = [] self._currentFilePath = None self._currentFileIterator = None self._currentFileOutput = [] self._currentFileErrors = [] self._dbgContext, self._profileName = context trace.cc(2, "Initialized new process: {0}".format(self.name)) #------------------------------------------------------------------------- def run(self): ''' Process entry point - set up debug/profile context ''' try: trace.set_context(self._dbgContext) if self._profileName is not None: import cProfile cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name) else: self._run() except Exception, e: self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e)) trace.traceback() except KeyboardInterrupt: trace.cc(1, "Ctrl-c occurred in job worker loop")
def run(self): trace.cc(1, "STARTING: Begining to process output queue...") try: if self._profileName is not None: import cProfile; cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name) else: self._run() trace.cc(1, "FINISHED processing output queue") except KeyboardInterrupt: trace.cc(1, "Ctrl-c occurred in OUTPUT THREAD") thread.interrupt_main() except Exception, e: trace.cc(1, "EXCEPTION occurred while processing output queue") self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e)) trace.traceback(2)
def _wait_then_exit(self): trace.cc(1, "Waiting to exit workers and output thread...") for worker in self._workers(): while worker.is_alive(): self._status_callback() worker.join(WORKER_EXIT_TIMEOUT) trace.cc(2, "Worker {0} is_alive: {1}".format( worker.name, worker.is_alive())) self._outThread.join(JOBOUT_EXIT_TIMEOUT) self._close_queues() trace.cc(1, "TERMINATING")
def _check_command(self): ''' Check the command queue to see if there have been problems while running a job Exceptions received from the command queue are thrown ''' myCommand = None otherCommands = [] try: while self._continueProcessing: (target, command, payload) = self._controlQueue.get_nowait() trace.cc(4, "check_command - {0}, {1}".format(target, command)) if target == 'JOB': if 'ERROR' == command: # Error notifications in the control queue are only used to support # break on error functionality -- the error info itself will be handled # by the output queue. Jobs with lots of errors can clog up the # control queue, so we clear these out as we find them trace.cc(1, "COMMAND: ERROR for file: {0}".format(payload)) if self._options.breakOnError: self._continueProcessing = False else: myCommand = command break else: otherCommands.append((target, command, payload)) except Empty: pass finally: if 'EXCEPTION' == myCommand: trace.cc(1, "COMMAND: EXCEPTION RECEIVED") raise payload # If everything is okay, or some other exception happened, we want to # make sure we put any queue items we removed back in the queue for (target, command, payload) in otherCommands: trace.cc(3, "putting {0}, {1}".format(target, command)) self._controlQueue.put_nowait((target, command, payload)) return self._continueProcessing
def _check_for_stop(self): ''' Command queue will normally be empty unless we are terminating. If there are commands, get all that we can until we find our own or empty the queue -- we then put back everything we took off. The command queue should never have a lot in it and this guarantees we can't deadlock. ''' otherCommands = [] myCommand = None exitNow = False try: while True: (target, command, payload) = self._controlQueue.get_nowait() trace.cc(3, "command - {0}, {1}".format(target, command)) if target == self.name: myCommand = command break else: otherCommands.append((target, command, payload)) except Empty: pass finally: if 'EXIT' == myCommand: trace.cc(1, "COMMAND: EXIT") self._continueProcessing = False exitNow = True elif 'WORK_DONE' == myCommand: trace.cc(1, "COMMAND: WORK_DONE") self._continueProcessing = False for target, command, payload in otherCommands: trace.cc(3, "putting {0}, {1}".format(target, command)) try: self._controlQueue.put((target, command, payload), True, CONTROL_QUEUE_TIMEOUT) except Full: raise utils.JobException( "FATAL EXCEPTION - Control Queue full, can't put") return exitNow
def _check_for_stop(self): ''' Command queue will normally be empty unless we are terminating. If there are commands, get all that we can until we find our own or empty the queue -- we then put back everything we took off. The command queue should never have a lot in it and this guarantees we can't deadlock. ''' otherCommands = [] myCommand = None exitNow = False try: while True: (target, command, payload) = self._controlQueue.get_nowait() trace.cc(3, "command - {0}, {1}".format(target, command)) if target == self.name: myCommand = command break else: otherCommands.append((target, command, payload)) except Empty: pass finally: if 'EXIT' == myCommand: trace.cc(1, "COMMAND: EXIT") self._continueProcessing = False exitNow = True elif 'WORK_DONE' == myCommand: trace.cc(1, "COMMAND: WORK_DONE") self._continueProcessing = False for target, command, payload in otherCommands: trace.cc(3, "putting {0}, {1}".format(target, command)) try: self._controlQueue.put((target, command, payload), True, CONTROL_QUEUE_TIMEOUT) except Full: raise utils.JobException("FATAL EXCEPTION - Control Queue full, can't put") return exitNow
try: trace.set_context(self._dbgContext) if self._profileName is not None: import cProfile; cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name) else: self._run() except Exception, e: self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e)) trace.traceback() except KeyboardInterrupt: trace.cc(1, "Ctrl-c occurred in job worker loop") except Exception, e: trace.cc(1, "EXCEPTION occurred in job worker loop") self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e)) trace.traceback() finally: # We know the input and out queues are empty or that we're bailing # on them, so we cancel_join_thread (don't wait for them to clear) self._inputQueue.close() self._inputQueue.cancel_join_thread() self._outputQueue.close() self._outputQueue.cancel_join_thread() # We may have put items on the control queue, so we join_thread to # make sure what we've put in the pipe is flushed self._controlQueue.close() self._controlQueue.join_thread() trace.cc(1, "TERMINATING")
def _keyboardInterrupt(self): trace.cc(1, "Ctrl-c occurred in MAIN loop") self._send_output_command('EXIT') raise
def _exception(self, e): trace.cc(1, "EXCEPTION -- EXITING JOB...") self._send_workers_command('EXIT') self._send_output_command('EXIT') trace.traceback() raise e
if self._profileName is not None: import cProfile; cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name) else: self._run() trace.cc(1, "FINISHED processing output queue") except KeyboardInterrupt: trace.cc(1, "Ctrl-c occurred in OUTPUT THREAD") thread.interrupt_main() except Exception, e: trace.cc(1, "EXCEPTION occurred while processing output queue") self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e)) trace.traceback(2) finally: trace.cc(1, "TERMINATING") def _run(self): # We keep processing queue until the job signals it is done and # the queue is empty, or we receive an abort command while self._continue_processing(): try: if self._workDone and self._outQueue.empty(): break filesOutput = self._outQueue.get_nowait() except Empty: trace.cc(3, "EMPTY OUTPUT") time.sleep(OUTPUT_EMPTY_WAIT) else:
def _send_command(self, target, command, payload): trace.cc(2, "COMMAND: {0}, {1} {2}".format(target, command, payload)) self._controlQueue.put_nowait((target, command, payload))
trace.set_context(self._dbgContext) if self._profileName is not None: import cProfile cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name) else: self._run() except Exception, e: self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e)) trace.traceback() except KeyboardInterrupt: trace.cc(1, "Ctrl-c occurred in job worker loop") except Exception, e: trace.cc(1, "EXCEPTION occurred in job worker loop") self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e)) trace.traceback() finally: # We know the input and out queues are empty or that we're bailing # on them, so we cancel_join_thread (don't wait for them to clear) self._inputQueue.close() self._inputQueue.cancel_join_thread() self._outputQueue.close() self._outputQueue.cancel_join_thread() # We may have put items on the control queue, so we join_thread to # make sure what we've put in the pipe is flushed self._controlQueue.close() self._controlQueue.join_thread() trace.cc(1, "TERMINATING")