def _fill_work_queue(self):
log.cc(1, "Starting to fill task queue...")
for pathToMeasure in self._pathsToMeasure:
if self._check_command():
self._folderWalker.walk(pathToMeasure)
if self._check_command() and self._workPackage.size_items() > 0:
self._send_current_package()
def _wait_output_finish(self):
log.cc(1, "Workers finished, waiting for output to finish...")
self._send_output_command('WORK_DONE')
while self._check_command() or self._outThread.is_alive():
self._outThread.join(JOB_EXIT_TIMEOUT)
self._status_callback()
self._continueProcessing = not bool(self._controlQueue.empty())
def _run(self):
# Keep processing queue until the job signals it is done and
# the queue is empty, or receive an abort command
while self._continue_processing():
try:
if self._workDone and self._outQueue.empty():
break
filesOutput = self._outQueue.get_nowait()
except Empty:
log.cc(3, "EMPTY OUTPUT")
time.sleep(OUTPUT_EMPTY_WAIT)
else:
self.taskPackagesReceived += 1
log.cc(2, "GOT {} measures".format(len(filesOutput)))
# Get a set of output for multiple files with each outputQueue item.
# Each file has a set of output and errors to pack up for app
for filePath, outputList, errorList in filesOutput:
# Synchronus callback to applicaiton
# Output writing and screen update occurs in this call
self._file_measure_callback(filePath, outputList,
errorList)
if errorList:
log.file(1, "ERROR measuring: {}".format(filePath))
self._controlQueue.put_nowait(
('JOB', 'ERROR', filePath))
def _continue_processing(self):
continueProcessing = True
otherCommands = []
myCommand = None
try:
while True:
(target, command, payload) = self._controlQueue.get_nowait()
log.cc(3, "command - {}, {}".format(target, command))
if target == self.name:
myCommand = command
break
else:
otherCommands.append((target, command, payload))
except Empty:
pass
finally:
if 'EXIT' == myCommand:
log.cc(2, "COMMAND: EXIT")
continueProcessing = False
elif 'WORK_DONE' == myCommand:
log.cc(2, "COMMAND: WORK_DONE")
self._workDone = True
if otherCommands:
log.cc(4, "replacing conmmands - {}".format(otherCommands))
utils.put_commands(self._controlQueue, otherCommands,
CONTROL_QUEUE_TIMEOUT)
return continueProcessing
def _send_current_package(self):
'''
Place package of work on queue, and start a worker
'''
self._workers.start_next()
log.cc(
2, "PUT WorkPackage - files: {}, bytes: {}...".format(
self._workPackage.size_items(),
self._workPackage.size_bytes()))
log.cc(4, list(self._workPackage.items()))
try:
self._taskQueue.put(list(self._workPackage.items()), True,
TASK_FULL_TIMEOUT)
except Full:
raise utils.JobException("FATAL ERROR -- FULL TASK QUEUE")
else:
self._taskPackagesSent += 1
self._filesSinceLastSend = 0
self._workPackage.reset()
def __init__(self, outQueue, controlQueue, profileName,
file_measure_callback):
log.cc(1, "Creating output queue thread")
threading.Thread.__init__(self, name="Out")
self._profileName = profileName
# The main thread manages our life; if something gets truly out of
# sync care more about exiting than ensuring output is flushed
self.daemon = True
# The main thread owns our queues
self._outQueue = outQueue
self._controlQueue = controlQueue
self._file_measure_callback = file_measure_callback
# Total task output packages we've received from all processes
self.taskPackagesReceived = 0
# Flag to track when receive WORK_DONE from the Job
self._workDone = False
def _put_files_in_queue(self, path, deltaPath, filesAndConfigs):
'''
Package files from the path into workItems that are grouped
into workPackages and placed into the task queue for jobworkers.
Packages are broken up if files number or total size exceeds
thresholds to help evenly distribute load across cores
'''
if not filesAndConfigs:
return
for fileName, configEntrys in filesAndConfigs:
# Expensive to check file size here, but worth it for pracelling widely
# varying file sizes out to cores for CPU intensive jobs.
# Profiling shows it is not worth caching this
try:
fileSize = utils.get_file_size(os.path.join(path, fileName))
except Exception as e:
# It is possible (at least in Windows) for a fileName to exist
# in the file system but be invalid for Windows calls. This is
# the first place the file is accessed through the file system;
# if it blows up don't want the job to fall apart, and this is
# an unusual case, so don't bother with a pathway back to the main
# application; just swallow it and provide debug
log.msg(1, str(e))
log.stack()
continue
log.cc(3, "WorkItem: {}, {}".format(fileSize, fileName))
self.numFilesToProcess += 1
workItem = (path, deltaPath, fileName, configEntrys, self._options,
len(filesAndConfigs))
self._workPackage.add(workItem, fileSize)
if self._workPackage.ready_to_send() or (self._filesSinceLastSend >
MAX_FILES_BEFORE_SEND):
self._send_current_package()
if not self._check_command():
break
def _check_command(self):
'''
Check command queue for any problems posted while running a job
Exceptions received from the command queue are unpackaged
and thrown for main to handle
'''
otherCommands = []
try:
while self._continueProcessing:
(target, command, payload) = self._controlQueue.get_nowait()
log.cc(4, "command check: {}, {}".format(target, command))
if target == 'JOB':
if 'ERROR' == command:
# Error notifications in the control queue are only used to support
# break on error -- the error info is handled by the output queue.
log.cc(1,
"COMMAND: ERROR for file: {}".format(payload))
if self._options.breakOnError:
self._continueProcessing = False
elif 'EXCEPTION' == command:
# Exceptions are bundled up for display to user
log.cc(1, "COMMAND: EXCEPTION RECEIVED")
if self._options.breakOnError:
self._continueProcessing = False
self.exceptions.append(payload)
else:
otherCommands.append((target, command, payload))
except Empty:
log.cc(4, "command check: empty")
finally:
# Put any queue items removed back in the queue
for (target, command, payload) in otherCommands:
log.cc(4, "command replace: {}, {}".format(target, command))
self._controlQueue.put_nowait((target, command, payload))
return self._continueProcessing
def run(self):
log.cc(1, "STARTING: Begining to process output queue...")
try:
if self._profileName is not None:
import cProfile
cProfile.runctx('self._run()', globals(), {'self': self},
self._profileName + self.name)
else:
self._run()
log.cc(1, "FINISHED processing output queue")
except KeyboardInterrupt:
log.cc(1, "Ctrl-c occurred in OUTPUT THREAD")
_thread.interrupt_main()
except Exception as e:
log.msg(1, "EXCEPTION processing output queue: " + str(e))
log.stack()
self._controlQueue.put_nowait(('JOB', 'EXCEPTION', e))
finally:
log.cc(1, "TERMINATING")
def _wait_then_exit(self):
log.cc(1, "Waiting to cleanup workers and output thread...")
self._send_workers_command('EXIT')
self._send_output_command('EXIT')
for worker in self._workers():
tries = 0
while worker.is_alive() and tries < WORKER_EXIT_TRIES:
self._status_callback()
worker.join(WORKER_EXIT_TIMEOUT)
log.cc(
2, "Worker {} is_alive: {}".format(worker.name,
worker.is_alive()))
self._check_command()
tries += 1
self._outThread.join(JOB_EXIT_TIMEOUT)
self._close_queues()
log.cc(1, "TERMINATING")
def _send_command(self, target, command, payload):
log.cc(2, "COMMAND: {}, {} {}".format(target, command, payload))
self._controlQueue.put_nowait((target, command, payload))
def _wait_process_packages(self):
log.cc(1, "Task queue complete, waiting for workers to finish...")
while self._check_command() and self._task_queue_size() > 0:
time.sleep(MAIN_PROCESSING_SLEEP)
self._status_callback()
log.cc(2, "Task queue size: " + str(self._task_queue_size()))