def __init__(self, processes=None, initializer=None, initargs=()):
self._inqueue = SimpleQueue()
self._outqueue = SimpleQueue()
self._taskqueue = Queue.Queue()
self._cache = {}
self._state = RUN
if processes is None:
try:
processes = processing.cpuCount()
except NotImplementedError:
processes = 1
self._pool = [
Process(target=worker, args=(self._inqueue, self._outqueue,
initializer, initargs))
for i in range(processes)
]
for i, w in enumerate(self._pool):
w.setName('PoolWorker-' + ':'.join(map(str, w._identity)))
w.start()
self._task_handler = threading.Thread(
target=Pool._handleTasks,
args=(self._taskqueue, self._inqueue, self._outqueue, self._pool)
)
self._task_handler.setDaemon(True)
self._task_handler._state = RUN
self._task_handler.start()
self._result_handler = threading.Thread(
target=Pool._handleResults,
args=(self._outqueue, self._cache)
)
self._result_handler.setDaemon(True)
self._result_handler._state = RUN
self._result_handler.start()
self._terminate = Finalize(
self, Pool._terminatePool,
args=(self._taskqueue, self._inqueue, self._outqueue,
self._cache, self._pool, self._task_handler,
self._result_handler),
exitpriority=5
)
def __init__(self, jobs, nprocesses=processing.cpuCount()):
"""
Parameters:
jobs - an FIFO list of instances of processing.Process that the
JobRunner will start
nprocesses - the number of processes to have concurrently running.
Default: the number of processors on the machine
"""
super(LocalJobRunner, self).__init__()
self._nprocesses = nprocesses
# make copy to use as a FIFO queue
self._queue = jobs[:]
self._queue.reverse()
self._running = []
self.times = {}
self.__finished = False
def __init__(self, jobs, nprocesses=processing.cpuCount()):
"""
Parameters:
jobs - an FIFO list of instances of processing.Process that the
JobRunner will start
nprocesses - the number of processes to have concurrently running.
Default: the number of processors on the machine
"""
super(LocalJobRunner, self).__init__()
self._nprocesses = nprocesses
# make copy to use as a FIFO queue
self._queue = jobs[:]
self._queue.reverse()
self._running = []
self.times = {}
self.__finished = False
def __init__(self, processes=None, initializer=None, initargs=()):
self._inqueue = SimpleQueue()
self._outqueue = SimpleQueue()
self._taskqueue = Queue.Queue()
self._cache = {}
self._state = RUN
if processes is None:
try:
processes = processing.cpuCount()
except NotImplementedError:
processes = 1
self._pool = [
Process(target=worker,
args=(self._inqueue, self._outqueue, initializer,
initargs)) for i in range(processes)
]
for i, w in enumerate(self._pool):
w.setName('PoolWorker-' + ':'.join(map(str, w._identity)))
w.start()
self._task_handler = threading.Thread(
target=Pool._handleTasks,
args=(self._taskqueue, self._inqueue, self._outqueue, self._pool))
self._task_handler.setDaemon(True)
self._task_handler._state = RUN
self._task_handler.start()
self._result_handler = threading.Thread(target=Pool._handleResults,
args=(self._outqueue,
self._cache))
self._result_handler.setDaemon(True)
self._result_handler._state = RUN
self._result_handler.start()
self._terminate = Finalize(
self,
Pool._terminatePool,
args=(self._taskqueue, self._inqueue, self._outqueue, self._cache,
self._pool, self._task_handler, self._result_handler),
exitpriority=5)
def test():
print 'cpuCount() = %d\n' % cpuCount()
#
# Create pool
#
PROCESSES = 4
print 'Creating pool with %d processes\n' % PROCESSES
pool = Pool(PROCESSES)
#
# Tests
#
TASKS = [(mul, (i, 7)) for i in range(10)] + \
[(plus, (i, 8)) for i in range(10)]
results = [pool.applyAsync(calculate, t) for t in TASKS]
imap_it = pool.imap(calculatestar, TASKS)
imap_unordered_it = pool.imapUnordered(calculatestar, TASKS)
print 'Ordered results using pool.applyAsync():'
for r in results:
print '\t', r.get()
print
print 'Ordered results using pool.imap():'
for x in imap_it:
print '\t', x
print
print 'Unordered results using pool.imapUnordered():'
for x in imap_unordered_it:
print '\t', x
print
print 'Ordered results using pool.map() --- will block till complete:'
for x in pool.map(calculatestar, TASKS):
print '\t', x
print
#
# Simple benchmarks
#
N = 100000
print 'def pow3(x): return x**3'
t = time.time()
A = map(pow3, xrange(N))
print '\tmap(pow3, xrange(%d)):\n\t\t%s seconds' % \
(N, time.time() - t)
t = time.time()
B = pool.map(pow3, xrange(N))
print '\tpool.map(pow3, xrange(%d)):\n\t\t%s seconds' % \
(N, time.time() - t)
t = time.time()
C = list(pool.imap(pow3, xrange(N), chunksize=N // 8))
print '\tlist(pool.imap(pow3, xrange(%d), chunksize=%d)):\n\t\t%s' \
' seconds' % (N, N//8, time.time() - t)
assert A == B == C, (len(A), len(B), len(C))
print
L = [None] * 1000000
print 'def noop(x): pass'
print 'L = [None] * 1000000'
t = time.time()
A = map(noop, L)
print '\tmap(noop, L):\n\t\t%s seconds' % \
(time.time() - t)
t = time.time()
B = pool.map(noop, L)
print '\tpool.map(noop, L):\n\t\t%s seconds' % \
(time.time() - t)
t = time.time()
C = list(pool.imap(noop, L, chunksize=len(L) // 8))
print '\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \
(len(L)//8, time.time() - t)
assert A == B == C, (len(A), len(B), len(C))
print
del A, B, C, L
#
# Test error handling
#
print 'Testing error handling:'
try:
print pool.apply(f, (5, ))
except ZeroDivisionError:
print '\tGot ZeroDivisionError as expected from pool.apply()'
else:
raise AssertionError, 'expected ZeroDivisionError'
try:
print pool.map(f, range(10))
except ZeroDivisionError:
print '\tGot ZeroDivisionError as expected from pool.map()'
else:
raise AssertionError, 'expected ZeroDivisionError'
try:
print list(pool.imap(f, range(10)))
except ZeroDivisionError:
print '\tGot ZeroDivisionError as expected from list(pool.imap())'
else:
raise AssertionError, 'expected ZeroDivisionError'
it = pool.imap(f, range(10))
for i in range(10):
try:
x = it.next()
except ZeroDivisionError:
if i == 5:
pass
except StopIteration:
break
else:
if i == 5:
raise AssertionError, 'expected ZeroDivisionError'
assert i == 9
print '\tGot ZeroDivisionError as expected from IMapIterator.next()'
print
#
# Testing timeouts
#
print 'Testing ApplyResult.get() with timeout:',
res = pool.applyAsync(calculate, TASKS[0])
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % res.get(0.02))
break
except TimeoutError:
sys.stdout.write('.')
print
print
print 'Testing IMapIterator.next() with timeout:',
it = pool.imap(calculatestar, TASKS)
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % it.next(0.02))
except StopIteration:
break
except TimeoutError:
sys.stdout.write('.')
print
print
#
# Testing callback
#
print 'Testing callback:'
A = []
B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729]
r = pool.applyAsync(mul, (7, 8), callback=A.append)
r.wait()
r = pool.mapAsync(pow3, range(10), callback=A.extend)
r.wait()
if A == B:
print '\tcallbacks succeeded\n'
else:
print '\t*** callbacks failed\n\t\t%s != %s\n' % (A, B)
#
# Check there are no outstanding tasks
#
assert not pool._cache, 'cache = %r' % pool._cache
#
# Check close() methods
#
print 'Testing close():'
for worker in pool._pool:
assert worker.isAlive()
result = pool.applyAsync(time.sleep, [0.5])
pool.close()
pool.join()
assert result.get() is None
for worker in pool._pool:
assert not worker.isAlive()
print '\tclose() succeeded\n'
#
# Check terminate() method
#
print 'Testing terminate():'
pool = Pool(2)
ignore = pool.apply(pow3, [2])
results = [pool.applyAsync(time.sleep, [10]) for i in range(10)]
pool.terminate()
pool.join()
for worker in pool._pool:
assert not worker.isAlive()
print '\tterminate() succeeded\n'
#
# Check garbage collection
#
print 'Testing garbage collection:'
pool = Pool(2)
processes = pool._pool
ignore = pool.apply(pow3, [2])
results = [pool.applyAsync(time.sleep, [10]) for i in range(10)]
del results, pool
time.sleep(0.2)
for worker in processes:
assert not worker.isAlive()
print '\tgarbage collection succeeded\n'
"""Support module for using the `multiprocessing` module."""
try:
import processing as multiprocessing
HAS_PROCESSING = True
except ImportError:
try:
import multiprocessing
HAS_PROCESSING = True
except ImportError:
HAS_PROCESSING = False
__all__ = ["use_parallel_processing"]
if HAS_PROCESSING:
CPU_COUNT = multiprocessing.cpuCount()
else:
CPU_COUNT = 1
multiprocessing = None
def use_parallel_processing():
"""Returns `True` if the query evaluator can run parallelized, `False` otherwise.
For the parallelized version, two conditions have to be fulfilled:
* the `multiprocessing` module is present
* the system has more than one CPU
"""
return HAS_PROCESSING and CPU_COUNT > 1
os.path.join(curpath, "config"),\
os.path.join(curpath, "app"))]
import setting
import json
import netutil
import uuid
import processing
from processing import Queue, Lock
import processing, logging
processing.enableLogging(level=logging.INFO)
# cpuCount
cpuCount = int(processing.cpuCount())
# min_space_left_in_giga
min_space_left_in_giga=setting.min_space_left_in_giga
# storage root
STORAGE_ROOT=setting.STORAGE_ROOT #{"/tudou/0":500, "/tudou/1":500}
dplayer_SERVER=setting.dplayer_SERVER
#
def check_disk(STORAGE_ROOT=None):
keys=STORAGE_ROOT.keys()
keys.sort()
return [STORAGE_ROOT[e] for e in keys]
import logging
log = logging.getLogger("dispatcher")
return np.sort(np.random.random(10000000))
'''
def f(x, N):
return cy_thread_test.cy_square(x, N)
def handleOutput(request, output):
"""This f'n, as a callback, is blocking.
It blocks the whole program, regardless of number of processes or CPUs/cores"""
print 'handleOutput got: %r, %r' % (request, output)
outputs.append(output)
#print 'pausing'
#for i in xrange(100000000):
# pass
if __name__ == '__main__':
ncpus = processing.cpuCount()
nthreads = ncpus + 1
print 'ncpus: %d, nthreads: %d' % (ncpus, nthreads)
pool = ThreadPool(nthreads) # create a threading pool
t0 = time.time()
#arr = np.random.random(10000000)
#for i, val in enumerate([1000000000]*10):#range(10):
for i in range(10):
args = (i, 1000000000)
print 'queueing task %d' % i
request = WorkRequest(f, args=args, callback=handleOutput)
# these requests will only multithread if f is a C extension call?? definitely don't multithread if f is pure Python
pool.putRequest(request)
print 'done queueing tasks'
pool.wait()
print 'tasks took %.3f sec' % time.time()
def f(x, N):
return cy_thread_test.cy_square(x, N)
def handleOutput(request, output):
"""This f'n, as a callback, is blocking.
It blocks the whole program, regardless of number of processes or CPUs/cores"""
print('handleOutput got: %r, %r' % (request, output))
outputs.append(output)
#print('pausing')
#for i in xrange(100000000):
# pass
if __name__ == '__main__':
ncpus = processing.cpuCount()
nthreads = ncpus + 1
print('ncpus: %d, nthreads: %d' % (ncpus, nthreads))
pool = ThreadPool(nthreads) # create a threading pool
t0 = time.time()
#arr = np.random.random(10000000)
#for i, val in enumerate([1000000000]*10):#range(10):
for i in range(10):
args = (i, 1000000000)
print('queueing task %d' % i)
request = WorkRequest(f, args=args, callback=handleOutput)
# these requests will only multithread if f is a C extension call?? definitely don't multithread if f is pure Python
pool.putRequest(request)
print('done queueing tasks')
pool.wait()
print('tasks took %.3f sec' % time.time())
def test():
print 'cpuCount() = %d\n' % cpuCount()
#
# Create pool
#
PROCESSES = 4
print 'Creating pool with %d processes\n' % PROCESSES
pool = Pool(PROCESSES)
#
# Tests
#
TASKS = [(mul, (i, 7)) for i in range(10)] + \
[(plus, (i, 8)) for i in range(10)]
results = [pool.applyAsync(calculate, t) for t in TASKS]
imap_it = pool.imap(calculatestar, TASKS)
imap_unordered_it = pool.imapUnordered(calculatestar, TASKS)
print 'Ordered results using pool.applyAsync():'
for r in results:
print '\t', r.get()
print
print 'Ordered results using pool.imap():'
for x in imap_it:
print '\t', x
print
print 'Unordered results using pool.imapUnordered():'
for x in imap_unordered_it:
print '\t', x
print
print 'Ordered results using pool.map() --- will block till complete:'
for x in pool.map(calculatestar, TASKS):
print '\t', x
print
#
# Simple benchmarks
#
N = 100000
print 'def pow3(x): return x**3'
t = time.time()
A = map(pow3, xrange(N))
print '\tmap(pow3, xrange(%d)):\n\t\t%s seconds' % \
(N, time.time() - t)
t = time.time()
B = pool.map(pow3, xrange(N))
print '\tpool.map(pow3, xrange(%d)):\n\t\t%s seconds' % \
(N, time.time() - t)
t = time.time()
C = list(pool.imap(pow3, xrange(N), chunksize=N//8))
print '\tlist(pool.imap(pow3, xrange(%d), chunksize=%d)):\n\t\t%s' \
' seconds' % (N, N//8, time.time() - t)
assert A == B == C, (len(A), len(B), len(C))
print
L = [None] * 1000000
print 'def noop(x): pass'
print 'L = [None] * 1000000'
t = time.time()
A = map(noop, L)
print '\tmap(noop, L):\n\t\t%s seconds' % \
(time.time() - t)
t = time.time()
B = pool.map(noop, L)
print '\tpool.map(noop, L):\n\t\t%s seconds' % \
(time.time() - t)
t = time.time()
C = list(pool.imap(noop, L, chunksize=len(L)//8))
print '\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \
(len(L)//8, time.time() - t)
assert A == B == C, (len(A), len(B), len(C))
print
del A, B, C, L
#
# Test error handling
#
print 'Testing error handling:'
try:
print pool.apply(f, (5,))
except ZeroDivisionError:
print '\tGot ZeroDivisionError as expected from pool.apply()'
else:
raise AssertionError, 'expected ZeroDivisionError'
try:
print pool.map(f, range(10))
except ZeroDivisionError:
print '\tGot ZeroDivisionError as expected from pool.map()'
else:
raise AssertionError, 'expected ZeroDivisionError'
try:
print list(pool.imap(f, range(10)))
except ZeroDivisionError:
print '\tGot ZeroDivisionError as expected from list(pool.imap())'
else:
raise AssertionError, 'expected ZeroDivisionError'
it = pool.imap(f, range(10))
for i in range(10):
try:
x = it.next()
except ZeroDivisionError:
if i == 5:
pass
except StopIteration:
break
else:
if i == 5:
raise AssertionError, 'expected ZeroDivisionError'
assert i == 9
print '\tGot ZeroDivisionError as expected from IMapIterator.next()'
print
#
# Testing timeouts
#
print 'Testing ApplyResult.get() with timeout:',
res = pool.applyAsync(calculate, TASKS[0])
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % res.get(0.02))
break
except TimeoutError:
sys.stdout.write('.')
print
print
print 'Testing IMapIterator.next() with timeout:',
it = pool.imap(calculatestar, TASKS)
while 1:
sys.stdout.flush()
try:
sys.stdout.write('\n\t%s' % it.next(0.02))
except StopIteration:
break
except TimeoutError:
sys.stdout.write('.')
print
print
#
# Testing callback
#
print 'Testing callback:'
A = []
B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729]
r = pool.applyAsync(mul, (7, 8), callback=A.append)
r.wait()
r = pool.mapAsync(pow3, range(10), callback=A.extend)
r.wait()
if A == B:
print '\tcallbacks succeeded\n'
else:
print '\t*** callbacks failed\n\t\t%s != %s\n' % (A, B)
#
# Check there are no outstanding tasks
#
assert not pool._cache, 'cache = %r' % pool._cache
#
# Check close() methods
#
print 'Testing close():'
for worker in pool._pool:
assert worker.isAlive()
result = pool.applyAsync(time.sleep, [0.5])
pool.close()
pool.join()
assert result.get() is None
for worker in pool._pool:
assert not worker.isAlive()
print '\tclose() succeeded\n'
#
# Check terminate() method
#
print 'Testing terminate():'
pool = Pool(2)
ignore = pool.apply(pow3, [2])
results = [pool.applyAsync(time.sleep, [10]) for i in range(10)]
pool.terminate()
pool.join()
for worker in pool._pool:
assert not worker.isAlive()
print '\tterminate() succeeded\n'
#
# Check garbage collection
#
print 'Testing garbage collection:'
pool = Pool(2)
processes = pool._pool
ignore = pool.apply(pow3, [2])
results = [pool.applyAsync(time.sleep, [10]) for i in range(10)]
del results, pool
time.sleep(0.2)
for worker in processes:
assert not worker.isAlive()
print '\tgarbage collection succeeded\n'
try:
import processing as multiprocessing
HAS_PROCESSING = True
except ImportError:
try:
import multiprocessing
HAS_PROCESSING = True
except ImportError:
HAS_PROCESSING = False
__all__ = ["use_parallel_processing"]
if HAS_PROCESSING:
CPU_COUNT = multiprocessing.cpuCount()
else:
CPU_COUNT = 1
multiprocessing = None
def use_parallel_processing():
"""Returns `True` if the query evaluator can run parallelized, `False` otherwise.
For the parallelized version, two conditions have to be fulfilled:
* the `multiprocessing` module is present
* the system has more than one CPU
"""
return HAS_PROCESSING and CPU_COUNT > 1
q = processing.Queue()
ps=[]
for i in range(4):
p = Process(target=work)
ps.append(p)
p.start()
print ps
for p in ps:
p.join()
print "We have %d CPUs" % processing.cpuCount()
pool = processing.Pool()
for i in ('f1','f2','f3','f4','f5'):
q.put(i)
while True:
try:
result = pool.apply_async(worker)
print result.get(timeout=2)
except Queue.Empty:
break
def worker():
file = q.get_nowait()
return 'worked on' + file
