def __init__(self, broker, init_func, init_args=(), init_kwargs=None, workers=None, max_worker_tasks=None, name=None, ): self.broker = broker self.init_func = init_func self.init_args = (broker.url,) + init_args self.init_kwargs = {} if init_kwargs is None else init_kwargs if workers is None: try: workers = cpu_count() except NotImplementedError: workers = 1 self.workers = workers self.max_worker_tasks = max_worker_tasks if name is None and broker.name != DEFAULT: name = broker.name self.name = name self._workers = [] self._worker_queue = ThreadQueue() self._running = False
def join(self): """Wait for pool to stop (call after ``.stop(join=False)``)""" assert not self._running, 'call stop() first' self._consumer_thread.join() self._worker_queue = ThreadQueue() for worker in self._workers: worker.join() log.info('%s stopped', str(self))
def score_bunch(self, scans, precursor_error_tolerance=1e-5, simplify=True, *args, **kwargs): # Map scans to target database self.log("... Querying Targets") waiting_task_results = ThreadQueue() def decoy_query_task(): self.log("... Querying Decoys") workload = self.decoy_evaluator._map_scans_to_hits( scans, precursor_error_tolerance) waiting_task_results.put(workload) workload = self.target_evaluator._map_scans_to_hits( scans, precursor_error_tolerance) # Execute the potentially disk-heavy task in the background while # evaluating the target spectrum matches. decoy_query_thread = threading.Thread(target=decoy_query_task) decoy_query_thread.start() # Evaluate mapped target hits target_solutions = [] workload_graph = workload.compute_workloads() total_work = workload.total_work_required(workload_graph) running_total_work = 0 for i, batch in enumerate(workload.batches(self.batch_size)): running_total_work += batch.batch_size self.log("... Batch %d (%d/%d) %0.2f%%" % (i + 1, running_total_work, total_work, (100. * running_total_work) / total_work)) target_scan_solution_map = self.target_evaluator._evaluate_hit_groups( batch, *args, **kwargs) # Aggregate and reduce target solutions temp = self.target_evaluator._collect_scan_solutions( target_scan_solution_map, batch.scan_map) temp = [case for case in temp if len(case) > 0] if simplify:
def __init__(self, subscribers=None, queue_limit=10): self.members = [] self.queue = ThreadQueue(queue_limit) for subscriber in subscribers or []: self.add(subscriber)
def __init__(self, handler): WrapperHandler.__init__(self, handler) self.queue = ThreadQueue(-1) self.controller = TWHThreadController(self) self.controller.start()
def fit(self, data): """Main thread: adds task while semaphore free, else blocks. Other thread is used to free up finished tasks. Quite simple to just Args: data (MicroArrayData): data. """ if self.verbose: print '[Parallel] fitting {} tasks with {} process{}...'.format( len(self.tasks), self.processes, 'es' if self.processes > 1 else '') assert issubclass(type(data), MicroArrayData) start_time = time.time() # need to use two different kinds of queues, one thread-safe and one process-safe task_queue = ThreadQueue() # Pipe tasks between threads result_queue = ProcessQueue() # Pipe results back to self.tasks list # keep track of start time per task def wrap_fit(task, data, index): """Wrapper of fit method, keep track of index of in self.task list where the results will be put back to """ result_queue.put((task.fit(data), index)) # Thread - start processes and acquire semaphore def add_processes(task_queue): indices = range(len(self.tasks)) if self.randomize: random.shuffle(indices) for index in indices: task = self.tasks[index] for _ in xrange(task.processes): self._semaphore.acquire() if self.verbose >= 3: time.sleep(0.1) print '[thread-start] acquired', task.processes, 'process{} for'.format( 'ses' if task.processes > 1 else ''), task.name p = Process(target=wrap_fit, args=(task, data, index)) # Need non-daemonic threads to use multiprocessed python processes. p.daemon = False p.start() # Put tuple of process and associated task in queue. task_queue.put((p, task)) task_queue.put(None) # send sentinal thread_add_processes = Thread(target=add_processes, args=(task_queue, )) thread_add_processes.start() # Thread - maintain processes and release semaphore def handle_processes(task_queue): running_tasks = [] finished = False print_count = 1 while not finished or len(running_tasks) > 0: # check task_queue at intervals if not task_queue.empty(): next_task = task_queue.get(timeout=0.1) # receive STOP sentinal, finish if next_task is None: finished = True else: running_tasks.append(next_task) # maintain process list; for proc, task in running_tasks[:]: if not proc.is_alive(): if self.verbose >= 3: print '[thread-maintain] releasing', task.processes, 'process{} for'.format( 'ses' if task.processes > 1 else ''), task.name for _ in xrange(task.processes): self._semaphore.release() proc.terminate() running_tasks.remove((proc, task)) break # need when a process is found that is done! time.sleep(.5) # print currently running processes every once in a while. if int((time.time() - start_time) / self.print_fitting_time ) > print_count and self.verbose >= 1: print '[Parallel][{:02d}h{:02d}m] running:'.format( *divmod(print_count * 10, 60)), for _, task in running_tasks: if task == running_tasks[-1][1]: # last task print '{}'.format(task.name) else: print '{},'.format(task.name), # print '[Parallel] {} ({:d}:{:2d})'.format(task.name, *divmod(int(start_time_task[task.name] - time.time()/60), 60)) print_count += 1 thread_handle_processes = Thread(target=handle_processes, args=(task_queue, )) thread_handle_processes.start() # Thread - catch results from result_queue and put back in self.task list def handle_results(): processed_results = 0 while processed_results < len(self.tasks): task, index = result_queue.get() if self.verbose >= 3: print '[thread-result] saving result for', task.name, 'to task list' self.tasks[index] = task processed_results += 1 time.sleep(.1) thread_handle_results = Thread(target=handle_results, args=()) thread_handle_results.start() # block main thread thread_add_processes.join() thread_handle_processes.join() thread_handle_results.join() assert all((i.done for i in self.tasks))
def __init__(self, *args): self.pid = 'not-started' self.queue = ThreadQueue() self.thread = Thread(target=self._proxy_loop, args=args) self.thread.start()
def create_queue(self, queue_limit): return ThreadQueue(queue_limit)
# unite! import logging from Queue import Queue as ThreadQueue from threading import Thread from datawire.core import app, db from datawire.processing.queue import make_consumer log = logging.getLogger(__name__) pool_queue = ThreadQueue(maxsize=1) def make_dispatcher(connection, queue, processor): def handle(body, message): pool_queue.put((processor, body, message), True) message.ack() make_consumer(connection, queue, handle) def worker_target(): while True: func, body, message = pool_queue.get(True) try: func(body, message) except Exception, e: log.exception(e) finally: pool_queue.task_done() db.session.close()