def __init__(self, broker,
init_func,
init_args=(),
init_kwargs=None,
workers=None,
max_worker_tasks=None,
name=None,
):
self.broker = broker
self.init_func = init_func
self.init_args = (broker.url,) + init_args
self.init_kwargs = {} if init_kwargs is None else init_kwargs
if workers is None:
try:
workers = cpu_count()
except NotImplementedError:
workers = 1
self.workers = workers
self.max_worker_tasks = max_worker_tasks
if name is None and broker.name != DEFAULT:
name = broker.name
self.name = name
self._workers = []
self._worker_queue = ThreadQueue()
self._running = False
def join(self):
"""Wait for pool to stop (call after ``.stop(join=False)``)"""
assert not self._running, 'call stop() first'
self._consumer_thread.join()
self._worker_queue = ThreadQueue()
for worker in self._workers:
worker.join()
log.info('%s stopped', str(self))
def score_bunch(self,
scans,
precursor_error_tolerance=1e-5,
simplify=True,
*args,
**kwargs):
# Map scans to target database
self.log("... Querying Targets")
waiting_task_results = ThreadQueue()
def decoy_query_task():
self.log("... Querying Decoys")
workload = self.decoy_evaluator._map_scans_to_hits(
scans, precursor_error_tolerance)
waiting_task_results.put(workload)
workload = self.target_evaluator._map_scans_to_hits(
scans, precursor_error_tolerance)
# Execute the potentially disk-heavy task in the background while
# evaluating the target spectrum matches.
decoy_query_thread = threading.Thread(target=decoy_query_task)
decoy_query_thread.start()
# Evaluate mapped target hits
target_solutions = []
workload_graph = workload.compute_workloads()
total_work = workload.total_work_required(workload_graph)
running_total_work = 0
for i, batch in enumerate(workload.batches(self.batch_size)):
running_total_work += batch.batch_size
self.log("... Batch %d (%d/%d) %0.2f%%" %
(i + 1, running_total_work, total_work,
(100. * running_total_work) / total_work))
target_scan_solution_map = self.target_evaluator._evaluate_hit_groups(
batch, *args, **kwargs)
# Aggregate and reduce target solutions
temp = self.target_evaluator._collect_scan_solutions(
target_scan_solution_map, batch.scan_map)
temp = [case for case in temp if len(case) > 0]
if simplify:
def __init__(self, subscribers=None, queue_limit=10):
self.members = []
self.queue = ThreadQueue(queue_limit)
for subscriber in subscribers or []:
self.add(subscriber)
def __init__(self, handler):
WrapperHandler.__init__(self, handler)
self.queue = ThreadQueue(-1)
self.controller = TWHThreadController(self)
self.controller.start()
def fit(self, data):
"""Main thread: adds task while semaphore free, else blocks.
Other thread is used to free up finished tasks. Quite simple to just
Args:
data (MicroArrayData): data.
"""
if self.verbose:
print '[Parallel] fitting {} tasks with {} process{}...'.format(
len(self.tasks), self.processes,
'es' if self.processes > 1 else '')
assert issubclass(type(data), MicroArrayData)
start_time = time.time()
# need to use two different kinds of queues, one thread-safe and one process-safe
task_queue = ThreadQueue() # Pipe tasks between threads
result_queue = ProcessQueue() # Pipe results back to self.tasks list
# keep track of start time per task
def wrap_fit(task, data, index):
"""Wrapper of fit method, keep track of index of in self.task
list where the results will be put back to
"""
result_queue.put((task.fit(data), index))
# Thread - start processes and acquire semaphore
def add_processes(task_queue):
indices = range(len(self.tasks))
if self.randomize: random.shuffle(indices)
for index in indices:
task = self.tasks[index]
for _ in xrange(task.processes):
self._semaphore.acquire()
if self.verbose >= 3:
time.sleep(0.1)
print '[thread-start] acquired', task.processes, 'process{} for'.format(
'ses' if task.processes > 1 else ''), task.name
p = Process(target=wrap_fit, args=(task, data, index))
# Need non-daemonic threads to use multiprocessed python processes.
p.daemon = False
p.start()
# Put tuple of process and associated task in queue.
task_queue.put((p, task))
task_queue.put(None) # send sentinal
thread_add_processes = Thread(target=add_processes,
args=(task_queue, ))
thread_add_processes.start()
# Thread - maintain processes and release semaphore
def handle_processes(task_queue):
running_tasks = []
finished = False
print_count = 1
while not finished or len(running_tasks) > 0:
# check task_queue at intervals
if not task_queue.empty():
next_task = task_queue.get(timeout=0.1)
# receive STOP sentinal, finish
if next_task is None:
finished = True
else:
running_tasks.append(next_task)
# maintain process list;
for proc, task in running_tasks[:]:
if not proc.is_alive():
if self.verbose >= 3:
print '[thread-maintain] releasing', task.processes, 'process{} for'.format(
'ses' if task.processes > 1 else ''), task.name
for _ in xrange(task.processes):
self._semaphore.release()
proc.terminate()
running_tasks.remove((proc, task))
break # need when a process is found that is done!
time.sleep(.5)
# print currently running processes every once in a while.
if int((time.time() - start_time) / self.print_fitting_time
) > print_count and self.verbose >= 1:
print '[Parallel][{:02d}h{:02d}m] running:'.format(
*divmod(print_count * 10, 60)),
for _, task in running_tasks:
if task == running_tasks[-1][1]: # last task
print '{}'.format(task.name)
else:
print '{},'.format(task.name),
# print '[Parallel] {} ({:d}:{:2d})'.format(task.name, *divmod(int(start_time_task[task.name] - time.time()/60), 60))
print_count += 1
thread_handle_processes = Thread(target=handle_processes,
args=(task_queue, ))
thread_handle_processes.start()
# Thread - catch results from result_queue and put back in self.task list
def handle_results():
processed_results = 0
while processed_results < len(self.tasks):
task, index = result_queue.get()
if self.verbose >= 3:
print '[thread-result] saving result for', task.name, 'to task list'
self.tasks[index] = task
processed_results += 1
time.sleep(.1)
thread_handle_results = Thread(target=handle_results, args=())
thread_handle_results.start()
# block main thread
thread_add_processes.join()
thread_handle_processes.join()
thread_handle_results.join()
assert all((i.done for i in self.tasks))
def __init__(self, *args):
self.pid = 'not-started'
self.queue = ThreadQueue()
self.thread = Thread(target=self._proxy_loop, args=args)
self.thread.start()
def create_queue(self, queue_limit):
return ThreadQueue(queue_limit)
# unite!
import logging
from Queue import Queue as ThreadQueue
from threading import Thread
from datawire.core import app, db
from datawire.processing.queue import make_consumer
log = logging.getLogger(__name__)
pool_queue = ThreadQueue(maxsize=1)
def make_dispatcher(connection, queue, processor):
def handle(body, message):
pool_queue.put((processor, body, message), True)
message.ack()
make_consumer(connection, queue, handle)
def worker_target():
while True:
func, body, message = pool_queue.get(True)
try:
func(body, message)
except Exception, e:
log.exception(e)
finally:
pool_queue.task_done()
db.session.close()
