def timing_worker(inqueue, outqueue, progressqueue,
initializer=None, initargs=(),
maxtasks=None):
'''
A modified worker thread that tracks how much CPU time is used.
'''
assert(maxtasks is None or (type(maxtasks) == int and maxtasks > 0))
put = outqueue.put
get = inqueue.get
if hasattr(inqueue, '_writer'):
inqueue._writer.close()
outqueue._reader.close()
if progressqueue is not None:
progressqueue._reader.close()
if initializer is not None:
initializer(*initargs)
mypid = os.getpid()
completed = 0
#t0 = time.time()
while maxtasks is None or (maxtasks and completed < maxtasks):
#print 'PID %i @ %f: get task' % (os.getpid(), time.time()-t0)
try:
# print 'Worker pid', os.getpid(), 'getting task'
task = get()
except (EOFError, IOError):
debug('worker pid ' + os.getpid() +
' got EOFError or IOError -- exiting')
break
except KeyboardInterrupt as e:
print('timing_worker caught KeyboardInterrupt during get()')
put((None, None, (None,(False,e))))
raise SystemExit('ctrl-c')
break
if task is None:
debug('worker got sentinel -- exiting')
break
# print 'PID %i @ %f: unpack task' % (os.getpid(), time.time()-t0)
job, i, func, args, kwds = task
if progressqueue is not None:
try:
# print 'Worker pid', os.getpid(), 'writing to progressqueue'
progressqueue.put((job, i, mypid))
except (EOFError, IOError):
print('worker got EOFError or IOError on progress queue -- exiting')
break
t1 = CpuMeas()
#print 'PID %i @ %f: run task' % (os.getpid(), time.time()-t0)
try:
success,val = (True, func(*args, **kwds))
except Exception as e:
success,val = (False, e)
#print 'timing_worker: caught', e
except KeyboardInterrupt as e:
success,val = (False, e)
#print 'timing_worker: caught ctrl-C during work', e
#print type(e)
put((None, None, (None,(False,e))))
raise
#print 'PID %i @ %f: ran task' % (os.getpid(), time.time()-t0)
t2 = CpuMeas()
dt = (t2.cpu_seconds_since(t1), t2.wall_seconds_since(t1))
put((job, i, dt,(success,val)))
completed += 1
#print 'PID %i @ %f: sent result' % (os.getpid(), time.time()-t0)
debug('worker exiting after %d tasks' % completed)
def __call__(self, *args, **kwargs):
t1 = CpuMeas()
R = self.func(*args, **kwargs)
t2 = CpuMeas()
dt = (t2.cpu_seconds_since(t1), t2.wall_seconds_since(t1))
return R,dt
def __init__(self, pool, pickleTraffic, nproc):
self.pool = pool
self.nproc = nproc
self.t = self.now(pickleTraffic)
self.cpu = CpuMeas()
def debug_worker(inqueue,
outqueue,
progressqueue,
initializer=None,
initargs=(),
maxtasks=None):
#print 'debug_worker()'
assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0)
put = outqueue.put
get = inqueue.get
if hasattr(inqueue, '_writer'):
inqueue._writer.close()
outqueue._reader.close()
progressqueue._reader.close()
if initializer is not None:
initializer(*initargs)
mypid = os.getpid()
completed = 0
while maxtasks is None or (maxtasks and completed < maxtasks):
#t0 = time.time()
try:
print 'Worker pid', os.getpid(), 'getting task'
task = get()
except (EOFError, IOError):
debug('worker got EOFError or IOError -- exiting')
print 'Worker pid', os.getpid(), 'got EOF/IOErr getting task'
break
except KeyboardInterrupt as e:
print 'debug_worker caught KeyboardInterrupt during get()'
put((None, None, (None, (False, e))))
raise SystemExit('ctrl-c')
break
if task is None:
debug('worker got sentinel -- exiting')
break
job, i, func, args, kwds = task
if progressqueue is not None:
try:
print 'Worker pid', os.getpid(), 'writing to progressqueue'
progressqueue.put((job, i, mypid))
except (EOFError, IOError):
print 'worker got EOFError or IOError on progress queue -- exiting'
break
t1 = CpuMeas()
try:
success, val = (True, func(*args, **kwds))
except Exception as e:
success, val = (False, e)
#print 'debug_worker: caught', e
except KeyboardInterrupt as e:
success, val = (False, e)
#print 'debug_worker: caught ctrl-C during work', e
#print type(e)
put((None, None, (None, (False, e))))
raise
t2 = CpuMeas()
dt = (t2.cpu_seconds_since(t1), t2.wall_seconds_since(t1))
put((job, i, dt, (success, val)))
completed += 1
debug('worker exiting after %d tasks' % completed)