def _fill_work_queue(self):
trace.cc(1, "Starting to fill task queue")
for pathToMeasure in self._pathsToMeasure:
if self._check_command():
self._folderWalker.walk(pathToMeasure)
if self._check_command() and self._workPackage.size_items() > 0:
self._send_current_package()
def _run(self):
# We keep processing queue until the job signals it is done and
# the queue is empty, or we receive an abort command
while self._continue_processing():
try:
if self._workDone and self._outQueue.empty():
break
filesOutput = self._outQueue.get_nowait()
except Empty:
trace.cc(3, "EMPTY OUTPUT")
time.sleep(OUTPUT_EMPTY_WAIT)
else:
self.taskPackagesReceived += 1
trace.cc(2, "GOT {0} measures".format(len(filesOutput)))
# We get a set of output for multiple files with each
# outputQueue item. Each file has a set of output
# and potential errors that we pack to the app
for filePath, outputList, errorList in filesOutput:
# Synchronus callback to applicaiton
# Output writing and screen update occurs in this call
self._file_measure_callback(filePath, outputList, errorList)
if errorList:
trace.file(1, "ERROR measuring: {0}".format(filePath))
self._controlQueue.put_nowait(('JOB', 'ERROR', filePath))
def _wait_process_packages(self):
trace.cc(1, "Task queue is complete, processing packages")
while self._check_command() and self._task_queue_size() > 0:
time.sleep(MAIN_PROCESSING_SLEEP)
self._status_callback()
if self._check_command():
self._send_workers_command('WORK_DONE')
else:
self._send_workers_command('EXIT')
def file_measured_callback(self, filePath, measures, analysisResults):
'''
Callback from the masurement module
We store up a list of tuples with the work output for a given file
'''
assert filePath == self._currentFilePath, "Measure callback out of sync"
trace.cc(3, "_file_measured_callback: {0}".format(filePath))
trace.file(3, " measures: {0}".format(measures))
trace.file(3, " analysis: {0}".format(analysisResults))
self._currentFileOutput.append((measures, analysisResults))
def file_measured_callback(self, filePath, measures, analysisResults):
'''
Callback from the masurement module
We store up a list of tuples with the work output for a given file
'''
assert filePath == self._currentFilePath, "Measure callback out of sync"
trace.cc(3, "_file_measured_callback: {0}".format(filePath))
trace.file(3, " measures: {0}".format(measures))
trace.file(3, " analysis: {0}".format(analysisResults))
self._currentFileOutput.append((measures, analysisResults))
def _file_complete(self):
'''
Cache the output from the measurement callbacks for current file
'''
if self._currentFileOutput or self._currentFileErrors:
self._currentOutput.append(
(self._currentFilePath, self._currentFileOutput, self._currentFileErrors))
trace.cc(3, "Caching results: {0}".format(self._currentFilePath))
else:
trace.cc(3, "No measures for: {0}".format(self._currentFilePath))
self._currentFileOutput = []
self._currentFileErrors = []
def _file_complete(self):
'''
Cache the output from the measurement callbacks for current file
'''
if self._currentFileOutput or self._currentFileErrors:
self._currentOutput.append(
(self._currentFilePath, self._currentFileOutput,
self._currentFileErrors))
trace.cc(3, "Caching results: {0}".format(self._currentFilePath))
else:
trace.cc(3, "No measures for: {0}".format(self._currentFilePath))
self._currentFileOutput = []
self._currentFileErrors = []
def _post_results(self):
'''
Send any cached results back to main process's out thread
This is a set of results for config measure of every file
in the last work package
'''
try:
self._outputQueue.put(self._currentOutput, True, OUT_PUT_TIMEOUT)
trace.cc(3, "OUT - PUT {0} items".format(len(self._currentOutput)))
except Full:
raise utils.JobException("FATAL EXCEPTION - Out Queue full, can't put")
finally:
self._currentOutput = []
def _continue_processing(self):
continueProcessing = True
otherCommands = []
myCommand = None
try:
while True:
(target, command, payload) = self._controlQueue.get_nowait()
trace.cc(3, "command - {0}, {1}".format(target, command))
if target == self.name:
myCommand = command
break
else:
otherCommands.append((target, command, payload))
except Empty:
pass
finally:
if 'EXIT' == myCommand:
trace.cc(1, "COMMAND: EXIT")
continueProcessing = False
elif 'WORK_DONE' == myCommand:
trace.cc(1, "COMMAND: WORK_DONE")
self._workDone = True
for (target, command, payload) in otherCommands:
trace.cc(3, "putting {0}, {1}".format(target, command))
try:
self._controlQueue.put((target, command, payload), True, CONTROL_QUEUE_TIMEOUT)
except Full:
raise utils.JobException("FATAL EXCEPTION - Control Queue full, can't put")
return continueProcessing
def _post_results(self):
'''
Send any cached results back to main process's out thread
This is a set of results for config measure of every file
in the last work package
'''
try:
self._outputQueue.put(self._currentOutput, True, OUT_PUT_TIMEOUT)
trace.cc(3, "OUT - PUT {0} items".format(len(self._currentOutput)))
except Full:
raise utils.JobException(
"FATAL EXCEPTION - Out Queue full, can't put")
finally:
self._currentOutput = []
def _send_current_package(self):
'''
Place package of work on queue, and start a worker
'''
self._workers.start_next()
trace.cc(2, "PUT WorkPackage - files: {0}, bytes: {1}...".format(
self._workPackage.size_items(), self._workPackage.size_bytes()))
trace.cc(4, self._workPackage.items())
try:
self._taskQueue.put(self._workPackage.items(), True, TASK_FULL_TIMEOUT)
except Full:
raise utils.JobException("FATAL ERROR -- FULL TASK QUEUE")
else:
self._taskPackagesSent += 1
self._filesSinceLastSend = 0
self._workPackage.reset()
def __init__(self, inputQueue, outputQueue, controlQueue,
context, num, jobName=WORKER_PROC_BASENAME):
'''
Init is called in the parent process
'''
multiprocessing.Process.__init__(self, name=jobName + str(num))
self._inputQueue = inputQueue
self._outputQueue = outputQueue
self._controlQueue = controlQueue
self._continueProcessing = True
self._currentOutput = []
self._currentFilePath = None
self._currentFileIterator = None
self._currentFileOutput = []
self._currentFileErrors = []
self._dbgContext, self._profileName = context
trace.cc(2, "Initialized new process: {0}".format(self.name))
def _put_files_in_queue(self, path, deltaPath, filesAndConfigs):
'''
Package files from the given folder into workItems that are then grouped
into workPackages that are placed into the task queue for jobworkers.
Packages are broken up if files number or total size exceeds
thresholds to help evenly distribute load across cores
'''
if not filesAndConfigs:
return
self.numFoldersMeasured += 1
for fileName, configEntrys in filesAndConfigs:
# Expensive to check file size here, but it is worth it for
# pracelling widely varying file sizes out to cores for CPU intensive
# jobs. Profiling shows it is not worth caching this
try:
fileSize = utils.get_file_size(os.path.join(path, fileName))
except Exception, e:
# It is possible (at least in Windows) for a fileName to exist
# in the file system but be invalid for Windows calls. This is
# the first place we try to access the file through the file
# system; if it blows up we don't want the job to fall apart,
# and this is an unusual case, so unlike more likely errors,
# we don't bother with a pathway back to the main application
# to handle the error; we just swallow it and provide debug
trace.msg(1, str(e))
continue
trace.cc(3, "WorkItem: {0}, {1}".format(fileSize, fileName))
self.numFilesToProcess += 1
workItem = (path,
deltaPath,
fileName,
configEntrys,
self._options,
len(filesAndConfigs))
self._workPackage.add(workItem, fileSize)
if self._workPackage.ready_to_send() or (
self._filesSinceLastSend > MAX_FILES_BEFORE_SEND):
self._send_current_package()
if not self._check_command():
break
def __init__(self, outQueue, controlQueue, profileName, file_measure_callback):
trace.cc(1, "Creating output queue thread")
threading.Thread.__init__(self, name="Out")
self._profileName = profileName
# The main thread manages our life; if something gets truly out of
# sync we care more about exiting than ensuring output is flushed
self.daemon = True
# The main thread owns our queues
self._outQueue = outQueue
self._controlQueue = controlQueue
self._file_measure_callback = file_measure_callback
# Total task output packages we've received from all processes
self.taskPackagesReceived = 0
# Flag to track when we receive WORK_DONE from the Job
self._workDone = False
def _run(self):
'''
Process items from input queue until it's empty and app signals all done
'''
trace.cc(1, "STARTING: Begining to process input queue...")
while self._continueProcessing:
try:
workPackage = self._inputQueue.get_nowait()
trace.cc(
2, "GOT WorkPackage - files: {0}".format(len(workPackage)))
except Empty:
# The input queue can return empty when it really isn't, or
# we are in mid job and have burned down the empty queue
# Sleeping after these helps performance with many cores, vs
# just blocking on inputQueue.get
trace.cc(3, "EMPTY INPUT")
time.sleep(INPUT_EMPTY_WAIT)
self._check_for_stop()
else:
for workItem in workPackage:
if not self._measure_file(workItem):
self._continueProcessing = False
break
self._post_results()
self._check_for_stop()
def _run(self):
'''
Process items from input queue until it's empty and app signals all done
'''
trace.cc(1, "STARTING: Begining to process input queue...")
while self._continueProcessing:
try:
workPackage = self._inputQueue.get_nowait()
trace.cc(2, "GOT WorkPackage - files: {0}".format(len(workPackage)))
except Empty:
# The input queue can return empty when it really isn't, or
# we are in mid job and have burned down the empty queue
# Sleeping after these helps performance with many cores, vs
# just blocking on inputQueue.get
trace.cc(3, "EMPTY INPUT")
time.sleep(INPUT_EMPTY_WAIT)
self._check_for_stop()
else:
for workItem in workPackage:
if not self._measure_file(workItem):
self._continueProcessing = False
break
self._post_results()
self._check_for_stop()
def __init__(self,
inputQueue,
outputQueue,
controlQueue,
context,
num,
jobName=WORKER_PROC_BASENAME):
'''
Init is called in the parent process
'''
multiprocessing.Process.__init__(self, name=jobName + str(num))
self._inputQueue = inputQueue
self._outputQueue = outputQueue
self._controlQueue = controlQueue
self._continueProcessing = True
self._currentOutput = []
self._currentFilePath = None
self._currentFileIterator = None
self._currentFileOutput = []
self._currentFileErrors = []
self._dbgContext, self._profileName = context
trace.cc(2, "Initialized new process: {0}".format(self.name))
class Worker(Process):
'''
The worker class executes as separate processes spawned by the Job
They take items from the input queue, delegate calls to the measurement
modules, and package measures for the output queue.
'''
def __init__(self,
inputQueue,
outputQueue,
controlQueue,
context,
num,
jobName=WORKER_PROC_BASENAME):
'''
Init is called in the parent process
'''
multiprocessing.Process.__init__(self, name=jobName + str(num))
self._inputQueue = inputQueue
self._outputQueue = outputQueue
self._controlQueue = controlQueue
self._continueProcessing = True
self._currentOutput = []
self._currentFilePath = None
self._currentFileIterator = None
self._currentFileOutput = []
self._currentFileErrors = []
self._dbgContext, self._profileName = context
trace.cc(2, "Initialized new process: {0}".format(self.name))
#-------------------------------------------------------------------------
def run(self):
'''
Process entry point - set up debug/profile context
'''
try:
trace.set_context(self._dbgContext)
if self._profileName is not None:
import cProfile
cProfile.runctx('self._run()', globals(), {'self': self},
self._profileName + self.name)
else:
self._run()
except Exception, e:
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
except KeyboardInterrupt:
trace.cc(1, "Ctrl-c occurred in job worker loop")
def run(self):
trace.cc(1, "STARTING: Begining to process output queue...")
try:
if self._profileName is not None:
import cProfile;
cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name)
else:
self._run()
trace.cc(1, "FINISHED processing output queue")
except KeyboardInterrupt:
trace.cc(1, "Ctrl-c occurred in OUTPUT THREAD")
thread.interrupt_main()
except Exception, e:
trace.cc(1, "EXCEPTION occurred while processing output queue")
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback(2)
def _wait_then_exit(self):
trace.cc(1, "Waiting to exit workers and output thread...")
for worker in self._workers():
while worker.is_alive():
self._status_callback()
worker.join(WORKER_EXIT_TIMEOUT)
trace.cc(2, "Worker {0} is_alive: {1}".format(
worker.name, worker.is_alive()))
self._outThread.join(JOBOUT_EXIT_TIMEOUT)
self._close_queues()
trace.cc(1, "TERMINATING")
def _check_command(self):
'''
Check the command queue to see if there have been problems while
running a job
Exceptions received from the command queue are thrown
'''
myCommand = None
otherCommands = []
try:
while self._continueProcessing:
(target, command, payload) = self._controlQueue.get_nowait()
trace.cc(4, "check_command - {0}, {1}".format(target, command))
if target == 'JOB':
if 'ERROR' == command:
# Error notifications in the control queue are only used to support
# break on error functionality -- the error info itself will be handled
# by the output queue. Jobs with lots of errors can clog up the
# control queue, so we clear these out as we find them
trace.cc(1, "COMMAND: ERROR for file: {0}".format(payload))
if self._options.breakOnError:
self._continueProcessing = False
else:
myCommand = command
break
else:
otherCommands.append((target, command, payload))
except Empty:
pass
finally:
if 'EXCEPTION' == myCommand:
trace.cc(1, "COMMAND: EXCEPTION RECEIVED")
raise payload
# If everything is okay, or some other exception happened, we want to
# make sure we put any queue items we removed back in the queue
for (target, command, payload) in otherCommands:
trace.cc(3, "putting {0}, {1}".format(target, command))
self._controlQueue.put_nowait((target, command, payload))
return self._continueProcessing
def _check_for_stop(self):
'''
Command queue will normally be empty unless we are terminating.
If there are commands, get all that we can until we find our own or empty
the queue -- we then put back everything we took off. The command queue
should never have a lot in it and this guarantees we can't deadlock.
'''
otherCommands = []
myCommand = None
exitNow = False
try:
while True:
(target, command, payload) = self._controlQueue.get_nowait()
trace.cc(3, "command - {0}, {1}".format(target, command))
if target == self.name:
myCommand = command
break
else:
otherCommands.append((target, command, payload))
except Empty:
pass
finally:
if 'EXIT' == myCommand:
trace.cc(1, "COMMAND: EXIT")
self._continueProcessing = False
exitNow = True
elif 'WORK_DONE' == myCommand:
trace.cc(1, "COMMAND: WORK_DONE")
self._continueProcessing = False
for target, command, payload in otherCommands:
trace.cc(3, "putting {0}, {1}".format(target, command))
try:
self._controlQueue.put((target, command, payload), True,
CONTROL_QUEUE_TIMEOUT)
except Full:
raise utils.JobException(
"FATAL EXCEPTION - Control Queue full, can't put")
return exitNow
def _check_for_stop(self):
'''
Command queue will normally be empty unless we are terminating.
If there are commands, get all that we can until we find our own or empty
the queue -- we then put back everything we took off. The command queue
should never have a lot in it and this guarantees we can't deadlock.
'''
otherCommands = []
myCommand = None
exitNow = False
try:
while True:
(target, command, payload) = self._controlQueue.get_nowait()
trace.cc(3, "command - {0}, {1}".format(target, command))
if target == self.name:
myCommand = command
break
else:
otherCommands.append((target, command, payload))
except Empty:
pass
finally:
if 'EXIT' == myCommand:
trace.cc(1, "COMMAND: EXIT")
self._continueProcessing = False
exitNow = True
elif 'WORK_DONE' == myCommand:
trace.cc(1, "COMMAND: WORK_DONE")
self._continueProcessing = False
for target, command, payload in otherCommands:
trace.cc(3, "putting {0}, {1}".format(target, command))
try:
self._controlQueue.put((target, command, payload), True, CONTROL_QUEUE_TIMEOUT)
except Full:
raise utils.JobException("FATAL EXCEPTION - Control Queue full, can't put")
return exitNow
try:
trace.set_context(self._dbgContext)
if self._profileName is not None:
import cProfile;
cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name)
else:
self._run()
except Exception, e:
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
except KeyboardInterrupt:
trace.cc(1, "Ctrl-c occurred in job worker loop")
except Exception, e:
trace.cc(1, "EXCEPTION occurred in job worker loop")
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
finally:
# We know the input and out queues are empty or that we're bailing
# on them, so we cancel_join_thread (don't wait for them to clear)
self._inputQueue.close()
self._inputQueue.cancel_join_thread()
self._outputQueue.close()
self._outputQueue.cancel_join_thread()
# We may have put items on the control queue, so we join_thread to
# make sure what we've put in the pipe is flushed
self._controlQueue.close()
self._controlQueue.join_thread()
trace.cc(1, "TERMINATING")
def _keyboardInterrupt(self):
trace.cc(1, "Ctrl-c occurred in MAIN loop")
self._send_output_command('EXIT')
raise
def _exception(self, e):
trace.cc(1, "EXCEPTION -- EXITING JOB...")
self._send_workers_command('EXIT')
self._send_output_command('EXIT')
trace.traceback()
raise e
if self._profileName is not None:
import cProfile;
cProfile.runctx('self._run()', globals(), {'self': self}, self._profileName + self.name)
else:
self._run()
trace.cc(1, "FINISHED processing output queue")
except KeyboardInterrupt:
trace.cc(1, "Ctrl-c occurred in OUTPUT THREAD")
thread.interrupt_main()
except Exception, e:
trace.cc(1, "EXCEPTION occurred while processing output queue")
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback(2)
finally:
trace.cc(1, "TERMINATING")
def _run(self):
# We keep processing queue until the job signals it is done and
# the queue is empty, or we receive an abort command
while self._continue_processing():
try:
if self._workDone and self._outQueue.empty():
break
filesOutput = self._outQueue.get_nowait()
except Empty:
trace.cc(3, "EMPTY OUTPUT")
time.sleep(OUTPUT_EMPTY_WAIT)
else:
def _send_command(self, target, command, payload):
trace.cc(2, "COMMAND: {0}, {1} {2}".format(target, command, payload))
self._controlQueue.put_nowait((target, command, payload))
trace.set_context(self._dbgContext)
if self._profileName is not None:
import cProfile
cProfile.runctx('self._run()', globals(), {'self': self},
self._profileName + self.name)
else:
self._run()
except Exception, e:
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
except KeyboardInterrupt:
trace.cc(1, "Ctrl-c occurred in job worker loop")
except Exception, e:
trace.cc(1, "EXCEPTION occurred in job worker loop")
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
trace.traceback()
finally:
# We know the input and out queues are empty or that we're bailing
# on them, so we cancel_join_thread (don't wait for them to clear)
self._inputQueue.close()
self._inputQueue.cancel_join_thread()
self._outputQueue.close()
self._outputQueue.cancel_join_thread()
# We may have put items on the control queue, so we join_thread to
# make sure what we've put in the pipe is flushed
self._controlQueue.close()
self._controlQueue.join_thread()
trace.cc(1, "TERMINATING")