def run(self): # Msg.user( "Worker Thread Instance Id: %s, (1)" % ( str( id( self ))) , "WORK-THREAD" ) my_sum_qitem = None try: self.setupWorkDir() my_launcher = self.create_launcher() Msg.user("Launcher Id 1: %s" % (str(id(my_launcher))), "WORK-THREAD") # Msg.user( "Worker Thread Instance Id: %s, (2)" % ( str( id( self ))) , "WORK-THREAD" ) my_launcher.launch() Msg.user("Launcher Id 2: %s" % (str(id(my_launcher))), "WORK-THREAD") # Msg.user( "Worker Thread Instance Id: %s, (3)" % ( str( id( self ))) , "WORK-THREAD" ) my_process_result = my_launcher.extract_results() Msg.user("Process Result: %s" % (my_process_result), "WORK-THREAD") Msg.user("Launcher Id 3: %s" % (str(id(my_launcher))), "WORK-THREAD") # self.launcher = my_launcher Msg.user("Process Result: %s" % (my_process_result), "WORK-THREAD") my_sum_qitem = SummaryQueueItem(my_process_result) Msg.user("Created Summary Queue Item", "WORK-THREAD") except Exception as arg_ex: Msg.error_trace(str(arg_ex)) Msg.err("Message: %s, Control File Path: %s" % (str(arg_ex), self.queue_item.work_dir)) my_sum_qitem = SummaryErrorQueueItem({ "error": arg_ex, "message": "Error Processing Task ...", "path": self.queue_item.ctrl_item.file_path(), "type": str(type(arg_ex)) }) finally: # my_launcher = None my_attempt = 0 while (self.summary.queue.enqueue(my_sum_qitem) == False): SysUtils.sleep(100) #heartbeat my_attempt += 1 if (my_attempt % 10) == 0: Msg.dbg("Attempt %d to insert into summary queue" % (my_attempt)) self.thread_count.delta(-1) Msg.user("Thread Count Decremented", "WORK-THREAD") self.done_semaphore.release() Msg.user("Semaphore Released", "WORK-THREAD") Msg.user("Launcher Id 5: %s" % (str(id(self.launcher))), "WORK-THREAD")
def run(self): try: while (not self.process_queue.fully_loaded) or ( self.process_queue.fully_loaded and (self.thread_count.value() > 0 or self.process_queue.size() > 0)): try: # if something tiggered a shutdown then exit the # processing loop and stop processing more items. if self.process_queue.summary.is_terminated(): Msg.user("Terminated", "PROCESS-THREAD") break # Pop off the top of the process queue (should block if # the queue is empty) my_queue_item = self.process_queue.dequeue(1) # Launch a thread with the item. Sempahore will block if # we've reached max workers Msg.user("Waiting for Semaphore ....", "PROCESS-THREAD") if my_queue_item.mAppsInfo.mMode == "count": Msg.user( "Counting %s" % my_queue_item.work_dir, "PROCESS-THREAD", ) my_queue_item.mAppsInfo.incrementTestCount() else: my_work_thread = ProcessWorkerThread( self.summary, self.semaphore, self.thread_count, my_queue_item, self.process_queue.processor_name, self.process_queue.process_cmd, self.process_queue.use_lsf(), ) except TimeoutError: pass finally: pass # need to wait until all worker threads are done before continuing while self.thread_count.value() > 0: Msg.user( "Thread Count: %d" % (self.thread_count.value()), "PROCESS-THREAD", ) SysUtils.sleep(100) finally: self.done_event.Signal()
def heartbeat(self): # increment the heartbeat and when debug messages are enabled then a heartbeat message will be # posted every self.heartbeat-interval ticks. Whenever the heartbeat method is called the current_tick # is updated self.current_tick += 1 if not bool(self.current_tick % self.heartbeat_rate): Msg.dbg("HiThread[%s] Still Working" % (self.name)) # the sleep in SysUtils uses milliseconds as does the rest of the computing world instead of # fractions of a second. Thus this will pause this thread for 10 seconds allowing the process # thread some processor time SysUtils.sleep(self.sleep_period) return False
def handle_signal(arg_signal, arg_stackframe): # it is necessary to write directly to stdout and not use print which is very unreliable if arg_signal == signal.SIGINT: sys.stdout.write( "Signal = {\'retcode\': %d, \'message\': \'Encountered interrupt, all processing halted\'}\n" % (signal.SIGINT)) elif arg_signal == signal.SIGTERM: sys.stdout.write( "Signal = {\'retcode\': %d, \'message\': \'OS Terminated Process, all processing halted\'}\n" % (signal.SIGTERM)) # Flush the line and release the processor to ensure that the output is fully written sys.stdout.flush() SysUtils.sleep(1) # once the line has been written kill any remaining processes dead dead dead, this will suppress further output os.killpg(0, signal.SIGKILL) # finally return the signal id as the return code sys.exit(int(arg_signal))