def __init__(self, processes=None, initializer=None, initargs=()): self._inqueue = SimpleQueue() self._outqueue = SimpleQueue() self._taskqueue = Queue.Queue() self._cache = {} self._state = RUN if processes is None: try: processes = processing.cpuCount() except NotImplementedError: processes = 1 self._pool = [ Process(target=worker, args=(self._inqueue, self._outqueue, initializer, initargs)) for i in range(processes) ] for i, w in enumerate(self._pool): w.setName('PoolWorker-' + ':'.join(map(str, w._identity))) w.start() self._task_handler = threading.Thread( target=Pool._handleTasks, args=(self._taskqueue, self._inqueue, self._outqueue, self._pool) ) self._task_handler.setDaemon(True) self._task_handler._state = RUN self._task_handler.start() self._result_handler = threading.Thread( target=Pool._handleResults, args=(self._outqueue, self._cache) ) self._result_handler.setDaemon(True) self._result_handler._state = RUN self._result_handler.start() self._terminate = Finalize( self, Pool._terminatePool, args=(self._taskqueue, self._inqueue, self._outqueue, self._cache, self._pool, self._task_handler, self._result_handler), exitpriority=5 )
def __init__(self, jobs, nprocesses=processing.cpuCount()): """ Parameters: jobs - an FIFO list of instances of processing.Process that the JobRunner will start nprocesses - the number of processes to have concurrently running. Default: the number of processors on the machine """ super(LocalJobRunner, self).__init__() self._nprocesses = nprocesses # make copy to use as a FIFO queue self._queue = jobs[:] self._queue.reverse() self._running = [] self.times = {} self.__finished = False
def __init__(self, processes=None, initializer=None, initargs=()): self._inqueue = SimpleQueue() self._outqueue = SimpleQueue() self._taskqueue = Queue.Queue() self._cache = {} self._state = RUN if processes is None: try: processes = processing.cpuCount() except NotImplementedError: processes = 1 self._pool = [ Process(target=worker, args=(self._inqueue, self._outqueue, initializer, initargs)) for i in range(processes) ] for i, w in enumerate(self._pool): w.setName('PoolWorker-' + ':'.join(map(str, w._identity))) w.start() self._task_handler = threading.Thread( target=Pool._handleTasks, args=(self._taskqueue, self._inqueue, self._outqueue, self._pool)) self._task_handler.setDaemon(True) self._task_handler._state = RUN self._task_handler.start() self._result_handler = threading.Thread(target=Pool._handleResults, args=(self._outqueue, self._cache)) self._result_handler.setDaemon(True) self._result_handler._state = RUN self._result_handler.start() self._terminate = Finalize( self, Pool._terminatePool, args=(self._taskqueue, self._inqueue, self._outqueue, self._cache, self._pool, self._task_handler, self._result_handler), exitpriority=5)
def test(): print 'cpuCount() = %d\n' % cpuCount() # # Create pool # PROCESSES = 4 print 'Creating pool with %d processes\n' % PROCESSES pool = Pool(PROCESSES) # # Tests # TASKS = [(mul, (i, 7)) for i in range(10)] + \ [(plus, (i, 8)) for i in range(10)] results = [pool.applyAsync(calculate, t) for t in TASKS] imap_it = pool.imap(calculatestar, TASKS) imap_unordered_it = pool.imapUnordered(calculatestar, TASKS) print 'Ordered results using pool.applyAsync():' for r in results: print '\t', r.get() print print 'Ordered results using pool.imap():' for x in imap_it: print '\t', x print print 'Unordered results using pool.imapUnordered():' for x in imap_unordered_it: print '\t', x print print 'Ordered results using pool.map() --- will block till complete:' for x in pool.map(calculatestar, TASKS): print '\t', x print # # Simple benchmarks # N = 100000 print 'def pow3(x): return x**3' t = time.time() A = map(pow3, xrange(N)) print '\tmap(pow3, xrange(%d)):\n\t\t%s seconds' % \ (N, time.time() - t) t = time.time() B = pool.map(pow3, xrange(N)) print '\tpool.map(pow3, xrange(%d)):\n\t\t%s seconds' % \ (N, time.time() - t) t = time.time() C = list(pool.imap(pow3, xrange(N), chunksize=N // 8)) print '\tlist(pool.imap(pow3, xrange(%d), chunksize=%d)):\n\t\t%s' \ ' seconds' % (N, N//8, time.time() - t) assert A == B == C, (len(A), len(B), len(C)) print L = [None] * 1000000 print 'def noop(x): pass' print 'L = [None] * 1000000' t = time.time() A = map(noop, L) print '\tmap(noop, L):\n\t\t%s seconds' % \ (time.time() - t) t = time.time() B = pool.map(noop, L) print '\tpool.map(noop, L):\n\t\t%s seconds' % \ (time.time() - t) t = time.time() C = list(pool.imap(noop, L, chunksize=len(L) // 8)) print '\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \ (len(L)//8, time.time() - t) assert A == B == C, (len(A), len(B), len(C)) print del A, B, C, L # # Test error handling # print 'Testing error handling:' try: print pool.apply(f, (5, )) except ZeroDivisionError: print '\tGot ZeroDivisionError as expected from pool.apply()' else: raise AssertionError, 'expected ZeroDivisionError' try: print pool.map(f, range(10)) except ZeroDivisionError: print '\tGot ZeroDivisionError as expected from pool.map()' else: raise AssertionError, 'expected ZeroDivisionError' try: print list(pool.imap(f, range(10))) except ZeroDivisionError: print '\tGot ZeroDivisionError as expected from list(pool.imap())' else: raise AssertionError, 'expected ZeroDivisionError' it = pool.imap(f, range(10)) for i in range(10): try: x = it.next() except ZeroDivisionError: if i == 5: pass except StopIteration: break else: if i == 5: raise AssertionError, 'expected ZeroDivisionError' assert i == 9 print '\tGot ZeroDivisionError as expected from IMapIterator.next()' print # # Testing timeouts # print 'Testing ApplyResult.get() with timeout:', res = pool.applyAsync(calculate, TASKS[0]) while 1: sys.stdout.flush() try: sys.stdout.write('\n\t%s' % res.get(0.02)) break except TimeoutError: sys.stdout.write('.') print print print 'Testing IMapIterator.next() with timeout:', it = pool.imap(calculatestar, TASKS) while 1: sys.stdout.flush() try: sys.stdout.write('\n\t%s' % it.next(0.02)) except StopIteration: break except TimeoutError: sys.stdout.write('.') print print # # Testing callback # print 'Testing callback:' A = [] B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729] r = pool.applyAsync(mul, (7, 8), callback=A.append) r.wait() r = pool.mapAsync(pow3, range(10), callback=A.extend) r.wait() if A == B: print '\tcallbacks succeeded\n' else: print '\t*** callbacks failed\n\t\t%s != %s\n' % (A, B) # # Check there are no outstanding tasks # assert not pool._cache, 'cache = %r' % pool._cache # # Check close() methods # print 'Testing close():' for worker in pool._pool: assert worker.isAlive() result = pool.applyAsync(time.sleep, [0.5]) pool.close() pool.join() assert result.get() is None for worker in pool._pool: assert not worker.isAlive() print '\tclose() succeeded\n' # # Check terminate() method # print 'Testing terminate():' pool = Pool(2) ignore = pool.apply(pow3, [2]) results = [pool.applyAsync(time.sleep, [10]) for i in range(10)] pool.terminate() pool.join() for worker in pool._pool: assert not worker.isAlive() print '\tterminate() succeeded\n' # # Check garbage collection # print 'Testing garbage collection:' pool = Pool(2) processes = pool._pool ignore = pool.apply(pow3, [2]) results = [pool.applyAsync(time.sleep, [10]) for i in range(10)] del results, pool time.sleep(0.2) for worker in processes: assert not worker.isAlive() print '\tgarbage collection succeeded\n'
"""Support module for using the `multiprocessing` module.""" try: import processing as multiprocessing HAS_PROCESSING = True except ImportError: try: import multiprocessing HAS_PROCESSING = True except ImportError: HAS_PROCESSING = False __all__ = ["use_parallel_processing"] if HAS_PROCESSING: CPU_COUNT = multiprocessing.cpuCount() else: CPU_COUNT = 1 multiprocessing = None def use_parallel_processing(): """Returns `True` if the query evaluator can run parallelized, `False` otherwise. For the parallelized version, two conditions have to be fulfilled: * the `multiprocessing` module is present * the system has more than one CPU """ return HAS_PROCESSING and CPU_COUNT > 1
os.path.join(curpath, "config"),\ os.path.join(curpath, "app"))] import setting import json import netutil import uuid import processing from processing import Queue, Lock import processing, logging processing.enableLogging(level=logging.INFO) # cpuCount cpuCount = int(processing.cpuCount()) # min_space_left_in_giga min_space_left_in_giga=setting.min_space_left_in_giga # storage root STORAGE_ROOT=setting.STORAGE_ROOT #{"/tudou/0":500, "/tudou/1":500} dplayer_SERVER=setting.dplayer_SERVER # def check_disk(STORAGE_ROOT=None): keys=STORAGE_ROOT.keys() keys.sort() return [STORAGE_ROOT[e] for e in keys] import logging log = logging.getLogger("dispatcher")
return np.sort(np.random.random(10000000)) ''' def f(x, N): return cy_thread_test.cy_square(x, N) def handleOutput(request, output): """This f'n, as a callback, is blocking. It blocks the whole program, regardless of number of processes or CPUs/cores""" print 'handleOutput got: %r, %r' % (request, output) outputs.append(output) #print 'pausing' #for i in xrange(100000000): # pass if __name__ == '__main__': ncpus = processing.cpuCount() nthreads = ncpus + 1 print 'ncpus: %d, nthreads: %d' % (ncpus, nthreads) pool = ThreadPool(nthreads) # create a threading pool t0 = time.time() #arr = np.random.random(10000000) #for i, val in enumerate([1000000000]*10):#range(10): for i in range(10): args = (i, 1000000000) print 'queueing task %d' % i request = WorkRequest(f, args=args, callback=handleOutput) # these requests will only multithread if f is a C extension call?? definitely don't multithread if f is pure Python pool.putRequest(request) print 'done queueing tasks' pool.wait() print 'tasks took %.3f sec' % time.time()
def f(x, N): return cy_thread_test.cy_square(x, N) def handleOutput(request, output): """This f'n, as a callback, is blocking. It blocks the whole program, regardless of number of processes or CPUs/cores""" print('handleOutput got: %r, %r' % (request, output)) outputs.append(output) #print('pausing') #for i in xrange(100000000): # pass if __name__ == '__main__': ncpus = processing.cpuCount() nthreads = ncpus + 1 print('ncpus: %d, nthreads: %d' % (ncpus, nthreads)) pool = ThreadPool(nthreads) # create a threading pool t0 = time.time() #arr = np.random.random(10000000) #for i, val in enumerate([1000000000]*10):#range(10): for i in range(10): args = (i, 1000000000) print('queueing task %d' % i) request = WorkRequest(f, args=args, callback=handleOutput) # these requests will only multithread if f is a C extension call?? definitely don't multithread if f is pure Python pool.putRequest(request) print('done queueing tasks') pool.wait() print('tasks took %.3f sec' % time.time())
def test(): print 'cpuCount() = %d\n' % cpuCount() # # Create pool # PROCESSES = 4 print 'Creating pool with %d processes\n' % PROCESSES pool = Pool(PROCESSES) # # Tests # TASKS = [(mul, (i, 7)) for i in range(10)] + \ [(plus, (i, 8)) for i in range(10)] results = [pool.applyAsync(calculate, t) for t in TASKS] imap_it = pool.imap(calculatestar, TASKS) imap_unordered_it = pool.imapUnordered(calculatestar, TASKS) print 'Ordered results using pool.applyAsync():' for r in results: print '\t', r.get() print print 'Ordered results using pool.imap():' for x in imap_it: print '\t', x print print 'Unordered results using pool.imapUnordered():' for x in imap_unordered_it: print '\t', x print print 'Ordered results using pool.map() --- will block till complete:' for x in pool.map(calculatestar, TASKS): print '\t', x print # # Simple benchmarks # N = 100000 print 'def pow3(x): return x**3' t = time.time() A = map(pow3, xrange(N)) print '\tmap(pow3, xrange(%d)):\n\t\t%s seconds' % \ (N, time.time() - t) t = time.time() B = pool.map(pow3, xrange(N)) print '\tpool.map(pow3, xrange(%d)):\n\t\t%s seconds' % \ (N, time.time() - t) t = time.time() C = list(pool.imap(pow3, xrange(N), chunksize=N//8)) print '\tlist(pool.imap(pow3, xrange(%d), chunksize=%d)):\n\t\t%s' \ ' seconds' % (N, N//8, time.time() - t) assert A == B == C, (len(A), len(B), len(C)) print L = [None] * 1000000 print 'def noop(x): pass' print 'L = [None] * 1000000' t = time.time() A = map(noop, L) print '\tmap(noop, L):\n\t\t%s seconds' % \ (time.time() - t) t = time.time() B = pool.map(noop, L) print '\tpool.map(noop, L):\n\t\t%s seconds' % \ (time.time() - t) t = time.time() C = list(pool.imap(noop, L, chunksize=len(L)//8)) print '\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \ (len(L)//8, time.time() - t) assert A == B == C, (len(A), len(B), len(C)) print del A, B, C, L # # Test error handling # print 'Testing error handling:' try: print pool.apply(f, (5,)) except ZeroDivisionError: print '\tGot ZeroDivisionError as expected from pool.apply()' else: raise AssertionError, 'expected ZeroDivisionError' try: print pool.map(f, range(10)) except ZeroDivisionError: print '\tGot ZeroDivisionError as expected from pool.map()' else: raise AssertionError, 'expected ZeroDivisionError' try: print list(pool.imap(f, range(10))) except ZeroDivisionError: print '\tGot ZeroDivisionError as expected from list(pool.imap())' else: raise AssertionError, 'expected ZeroDivisionError' it = pool.imap(f, range(10)) for i in range(10): try: x = it.next() except ZeroDivisionError: if i == 5: pass except StopIteration: break else: if i == 5: raise AssertionError, 'expected ZeroDivisionError' assert i == 9 print '\tGot ZeroDivisionError as expected from IMapIterator.next()' print # # Testing timeouts # print 'Testing ApplyResult.get() with timeout:', res = pool.applyAsync(calculate, TASKS[0]) while 1: sys.stdout.flush() try: sys.stdout.write('\n\t%s' % res.get(0.02)) break except TimeoutError: sys.stdout.write('.') print print print 'Testing IMapIterator.next() with timeout:', it = pool.imap(calculatestar, TASKS) while 1: sys.stdout.flush() try: sys.stdout.write('\n\t%s' % it.next(0.02)) except StopIteration: break except TimeoutError: sys.stdout.write('.') print print # # Testing callback # print 'Testing callback:' A = [] B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729] r = pool.applyAsync(mul, (7, 8), callback=A.append) r.wait() r = pool.mapAsync(pow3, range(10), callback=A.extend) r.wait() if A == B: print '\tcallbacks succeeded\n' else: print '\t*** callbacks failed\n\t\t%s != %s\n' % (A, B) # # Check there are no outstanding tasks # assert not pool._cache, 'cache = %r' % pool._cache # # Check close() methods # print 'Testing close():' for worker in pool._pool: assert worker.isAlive() result = pool.applyAsync(time.sleep, [0.5]) pool.close() pool.join() assert result.get() is None for worker in pool._pool: assert not worker.isAlive() print '\tclose() succeeded\n' # # Check terminate() method # print 'Testing terminate():' pool = Pool(2) ignore = pool.apply(pow3, [2]) results = [pool.applyAsync(time.sleep, [10]) for i in range(10)] pool.terminate() pool.join() for worker in pool._pool: assert not worker.isAlive() print '\tterminate() succeeded\n' # # Check garbage collection # print 'Testing garbage collection:' pool = Pool(2) processes = pool._pool ignore = pool.apply(pow3, [2]) results = [pool.applyAsync(time.sleep, [10]) for i in range(10)] del results, pool time.sleep(0.2) for worker in processes: assert not worker.isAlive() print '\tgarbage collection succeeded\n'
try: import processing as multiprocessing HAS_PROCESSING = True except ImportError: try: import multiprocessing HAS_PROCESSING = True except ImportError: HAS_PROCESSING = False __all__ = ["use_parallel_processing"] if HAS_PROCESSING: CPU_COUNT = multiprocessing.cpuCount() else: CPU_COUNT = 1 multiprocessing = None def use_parallel_processing(): """Returns `True` if the query evaluator can run parallelized, `False` otherwise. For the parallelized version, two conditions have to be fulfilled: * the `multiprocessing` module is present * the system has more than one CPU """ return HAS_PROCESSING and CPU_COUNT > 1
q = processing.Queue() ps=[] for i in range(4): p = Process(target=work) ps.append(p) p.start() print ps for p in ps: p.join() print "We have %d CPUs" % processing.cpuCount() pool = processing.Pool() for i in ('f1','f2','f3','f4','f5'): q.put(i) while True: try: result = pool.apply_async(worker) print result.get(timeout=2) except Queue.Empty: break def worker(): file = q.get_nowait() return 'worked on' + file