def close(self): debug('closing pool') if self._state == RUN: self._state = CLOSE self._worker_handler.close() self._worker_handler.join() self._taskqueue.put(None)
def worker(inqueue, outqueue, initializer=None, initargs=()): put = outqueue.put get = inqueue.get if hasattr(inqueue, '_writer'): inqueue._writer.close() outqueue._reader.close() if initializer is not None: initializer(*initargs) while 1: try: task = get() except (EOFError, IOError): debug('worker got EOFError or IOError -- exiting') break if task is None: debug('worker got sentinel -- exiting') break job, i, func, args, kwds = task try: result = (True, func(*args, **kwds)) except Exception as e: result = (False, e) put((job, i, result))
def temp(self, *args, **kwds): util.debug('requesting creation of a shared %r object', typeid) token, exp = self._create(typeid, *args, **kwds) proxy = proxytype(token, self._serializer, manager=self, authkey=self._authkey, exposed=exp) conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy
def _join_exited_workers(self, lost_worker_timeout=10.0): """Cleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. """ now = None # The worker may have published a result before being terminated, # but we have no way to accurately tell if it did. So we wait for # 10 seconds before we mark the job with WorkerLostError. for job in [job for job in self._cache.values() if not job.ready() and job._worker_lost]: now = now or time.time() if now - job._worker_lost > lost_worker_timeout: err = WorkerLostError("Worker exited prematurely.") job._set(None, (False, err)) cleaned = [] for i in reversed(range(len(self._pool))): worker = self._pool[i] if worker.exitcode is not None: # worker exited debug('cleaning up worker %d' % i) worker.join() cleaned.append(worker.pid) del self._pool[i] if cleaned: for job in self._cache.values(): for worker_pid in job.worker_pids(): if worker_pid in cleaned and not job.ready(): if self._putlock is not None: self._putlock.release() job._worker_lost = time.time() continue return True return False
def _join_exited_workers(self): """Cleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. """ cleaned = [] for i in reversed(range(len(self._pool))): worker = self._pool[i] if worker.exitcode is not None: # worker exited debug('cleaning up worker %d' % i) worker.join() cleaned.append(worker.pid) del self._pool[i] if cleaned: for job in self._cache.values(): for worker_pid in job.worker_pids(): if worker_pid in cleaned and not job.ready(): if self._putlock is not None: try: self._putlock.release() except Exception: pass err = WorkerLostError("Worker exited prematurely.") job._set(None, (False, err)) continue return True return False
def _feed(buffer, notempty, send, writelock, close, ignore_epipe): debug('starting thread to feed data to pipe') from .util import is_exiting nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != 'win32': wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None try: while 1: nacquire() try: if not buffer: nwait() finally: nrelease() try: while 1: obj = bpopleft() if obj is sentinel: debug('feeder thread got sentinel -- exiting') close() return if wacquire is None: send(obj) # Delete references to object. See issue16284 del obj else: wacquire() try: send(obj) # Delete references to object. See issue16284 del obj finally: wrelease() except IndexError: pass except Exception as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. try: if is_exiting(): info('error in queue thread: %s', e) else: import traceback traceback.print_exc() except Exception: pass
def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None): assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) put = outqueue.put get = inqueue.get if hasattr(inqueue, '_writer'): inqueue._writer.close() outqueue._reader.close() if initializer is not None: initializer(*initargs) completed = 0 while maxtasks is None or (maxtasks and completed < maxtasks): try: task = get() except (EOFError, IOError): debug('worker got EOFError or IOError -- exiting') break if task is None: debug('worker got sentinel -- exiting') break job, i, func, args, kwds = task try: result = (True, func(*args, **kwds)) except Exception, e: result = (False, e) try: put((job, i, result)) except Exception as e: wrapped = create_detailed_pickling_error(e, result[1]) put((job, i, (False, wrapped))) completed += 1
def SocketClient(address): ''' Return a connection object connected to the socket given by `address` ''' family = address_type(address) with socket.socket( getattr(socket, family) ) as s: s.setblocking(True) t = _init_timeout() while 1: try: s.connect(address) except socket.error as e: if e.args[0] != errno.ECONNREFUSED or _check_timeout(t): debug('failed to connect to address %s', address) raise time.sleep(0.01) else: break else: raise fd = duplicate(s.fileno()) conn = _multiprocessing.Connection(fd) return conn
def cancel_join_thread(self): debug('Queue.cancel_join_thread()') self._joincancelled = True try: self._jointhread.cancel() except AttributeError: pass
def _join_exited_workers(self): """Cleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. """ cleaned = False for i in reversed(range(len(self._pool))): worker = self._pool[i] if worker.exitcode is not None: # worker exited try: worker.join() except RuntimeError: # # RuntimeError: cannot join thread before it is started # # This is a race condition in DaemonProcess.start() which was found # during some of the test scans I run. The race condition exists # because we're using Threads for a Pool that was designed to be # used with real processes: thus there is no worker.exitcode, # thus it has to be simulated in a race condition-prone way. # continue else: debug('cleaning up worker %d' % i) cleaned = True del self._pool[i] return cleaned
def __init__(self, kind, value, maxvalue): # unlink_now is only used on win32 or when we are using fork. unlink_now = False for i in range(100): try: self._semlock = _SemLock( kind, value, maxvalue, SemLock._make_name(), unlink_now) except FileExistsError: # pragma: no cover pass else: break else: # pragma: no cover raise FileExistsError('cannot find name for semaphore') util.debug('created semlock with handle %s and name "%s"' % (self._semlock.handle, self._semlock.name)) self._make_methods() def _after_fork(obj): obj._semlock._after_fork() util.register_after_fork(self, _after_fork) # When the object is garbage collected or the # process shuts down we unlink the semaphore name semaphore_tracker.register(self._semlock.name) util.Finalize(self, SemLock._cleanup, (self._semlock.name,), exitpriority=0)
def _handle_workers(pool): while pool._worker_handler._state == RUN and pool._state == RUN: pool._maintain_pool() time.sleep(0.1) # send sentinel to stop workers pool._taskqueue.put(None) debug('worker handler exiting')
def join(self): debug('joining pool') assert self._state in (CLOSE, TERMINATE) self._task_handler.join() self._result_handler.join() for p in self._pool: p.join()
def _callmethod(self, methodname, args=(), kwds={}): ''' Try to call a method of the referrent and return a copy of the result ''' try: conn = self._tls.connection except AttributeError: util.debug('thread %r does not own a connection', threading.current_thread().name) self._connect() conn = self._tls.connection conn.send((self._id, methodname, args, kwds)) kind, result = conn.recv() if kind == '#RETURN': return result elif kind == '#PROXY': exposed, token = result proxytype = self._manager._registry[token.typeid][-1] token.address = self._token.address proxy = proxytype( token, self._serializer, manager=self._manager, authkey=self._authkey, exposed=exposed ) conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy raise convert_to_error(kind, result)
def create(self, c, typeid, *args, **kwds): """ Create a new shared object and return its id """ self.mutex.acquire() try: callable, exposed, method_to_typeid, proxytype = self.registry[typeid] if callable is None: if not (len(args) == 1 and not kwds): raise AssertionError obj = args[0] else: obj = callable(*args, **kwds) exposed = exposed is None and public_methods(obj) if method_to_typeid is not None: if not type(method_to_typeid) is dict: raise AssertionError exposed = list(exposed) + list(method_to_typeid) ident = '%x' % id(obj) util.debug('%r callable returned object with id %r', typeid, ident) self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid) self.id_to_refcount[ident] = ident not in self.id_to_refcount and 0 self.incref(c, ident) return (ident, tuple(exposed)) finally: self.mutex.release() return
def _help_stuff_finish(inqueue, task_handler, size): # task_handler may be blocked trying to put items on inqueue debug('removing tasks from inqueue until task handler finished') inqueue._rlock.acquire() while task_handler.is_alive() and inqueue._reader.poll(): inqueue._reader.recv() time.sleep(0)
def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None): pid = os.getpid() assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) put = outqueue.put get = inqueue.get if hasattr(inqueue, '_reader'): def poll(timeout): if inqueue._reader.poll(timeout): return True, get() return False, None else: def poll(timeout): # noqa try: return True, get(timeout=timeout) except Queue.Empty: return False, None if hasattr(inqueue, '_writer'): inqueue._writer.close() outqueue._reader.close() if initializer is not None: initializer(*initargs) if SIG_SOFT_TIMEOUT is not None: signal.signal(SIG_SOFT_TIMEOUT, soft_timeout_sighandler) completed = 0 while maxtasks is None or (maxtasks and completed < maxtasks): try: ready, task = poll(1.0) if not ready: continue except (EOFError, IOError): debug('worker got EOFError or IOError -- exiting') break if task is None: debug('worker got sentinel -- exiting') break job, i, func, args, kwds = task put((ACK, (job, i, time.time(), pid))) try: result = (True, func(*args, **kwds)) except Exception: result = (False, ExceptionInfo(sys.exc_info())) try: put((READY, (job, i, result))) except Exception, exc: _, _, tb = sys.exc_info() wrapped = MaybeEncodingError(exc, result[1]) einfo = ExceptionInfo((MaybeEncodingError, wrapped, tb)) put((READY, (job, i, (False, einfo)))) completed += 1
def _handle_workers(pool): thread = threading.current_thread() while thread._state == RUN or pool._cache and thread._state != TERMINATE: pool._maintain_pool() time.sleep(0.1) pool._taskqueue.put(None) debug('worker handler exiting')
def join(self): assert self._state in (CLOSE, TERMINATE) self._worker_handler.join() self._task_handler.join() self._result_handler.join() for p in self._pool: p.join() debug('after join()')
def _repopulate_pool(self): """Bring the number of pool processes up to the specified number, for use after reaping workers which have exited. """ debug('repopulating pool') for i in range(self._processes - len(self._pool)): self._create_worker_process() debug('added worker')
def _connect(self): util.debug('making connection to manager') name = current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name conn = self._Client(self._token.address, authkey=self._authkey) dispatch(conn, None, 'accept_connection', (name,)) self._tls.connection = conn
def _finalize_close(buffer, notempty): debug('telling queue thread to quit') notempty.acquire() try: buffer.append(_sentinel) notempty.notify() finally: notempty.release()
def join(self): debug('joining pool') raise self._state in (CLOSE, TERMINATE) or AssertionError self._worker_handler.join() self._task_handler.join() self._result_handler.join() for p in self._pool: p.join()
def _feed(buffer, notempty, send_bytes, writelock, close, reducers, ignore_epipe, onerror, queue_sem): util.debug('starting thread to feed data to pipe') nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != 'win32': wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None while 1: try: nacquire() try: if not buffer: nwait() finally: nrelease() try: while 1: obj = bpopleft() if obj is sentinel: util.debug('feeder thread got sentinel -- exiting') close() return # serialize the data before acquiring the lock obj_ = CustomizableLokyPickler.dumps( obj, reducers=reducers) if wacquire is None: send_bytes(obj_) else: wacquire() try: send_bytes(obj_) finally: wrelease() # Remove references early to avoid leaking memory del obj, obj_ except IndexError: pass except BaseException as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. if util.is_exiting(): util.info('error in queue thread: %s', e) return else: queue_sem.release() onerror(e, obj)
def _repopulate_pool(self): """Bring the number of pool processes up to the specified number, for use after reaping workers which have exited. """ for i in range(self._processes - len(self._pool)): if self._state != RUN: return self._create_worker_process() debug("added worker")
def close(self): debug("closing pool") if self._state == RUN: self._state = CLOSE self._worker_handler.close() self._worker_handler.join() self._taskqueue.put(None) if self._putlock: self._putlock.clear()
def __init__(self, kind, value, maxvalue): sl = self._semlock = _multiprocessing.SemLock(kind, value, maxvalue) debug('created semlock with handle %s' % sl.handle) self._make_methods() if sys.platform != 'win32': def _after_fork(obj): obj._semlock._after_fork() register_after_fork(self, _after_fork)
def _repopulate_pool(self): for i in range(self._processes - len(self._pool)): w = self.Process(target=worker, args=(self._inqueue, self._outqueue, self._initializer, self._initargs, self._terminator, self._maxtasksperchild)) self._pool.append(w) w.name = w.name.replace('Process', 'PoolWorker') w.daemon = True w.start() debug('added worker')
def serve_client(self, conn): util.debug('starting server thread to service %r', threading.current_thread().name) recv = conn.recv send = conn.send id_to_obj = self.id_to_obj while not self.stop: try: methodname = obj = None request = recv() ident, methodname, args, kwds = request obj, exposed, gettypeid = id_to_obj[ident] if methodname not in exposed: raise AttributeError('method %r of %r object is not in exposed=%r' % (methodname, type(obj), exposed)) function = getattr(obj, methodname) try: res = function(*args, **kwds) except Exception as e: msg = ('#ERROR', e) else: typeid = gettypeid and gettypeid.get(methodname, None) if typeid: rident, rexposed = self.create(conn, typeid, res) token = Token(typeid, self.address, rident) msg = ('#PROXY', (rexposed, token)) else: msg = ('#RETURN', res) except AttributeError: if methodname is None: msg = ('#TRACEBACK', format_exc()) else: try: fallback_func = self.fallback_mapping[methodname] result = fallback_func(self, conn, ident, obj, *args, **kwds) msg = ('#RETURN', result) except Exception: msg = ('#TRACEBACK', format_exc()) except EOFError: util.debug('got EOF -- exiting thread serving %r', threading.current_thread().name) sys.exit(0) except Exception: msg = ('#TRACEBACK', format_exc()) try: try: send(msg) except Exception as e: send(('#UNSERIALIZABLE', repr(msg))) except Exception as e: util.info('exception in thread serving %r', threading.current_thread().name) util.info(' ... message was %r', msg) util.info(' ... exception was %r', e) conn.close() sys.exit(1)
def manage_tasks(self, tasks): """Receives completed tasks from the clients and updates the tasks file. Args: tasks (dict): Tasks. """ t0 = Time.time() tasks_pth = path(self.params().get("tasks_pth", "")) n_tasks = len(tasks) n_done0 = n_tasks - self.taskq().qsize() n_done = n_done0 util.info("[Server] Tasks queued. {}/{} ({:.1f}%%) complete.".format( n_done, n_tasks, n_done / n_tasks * 100)) save_time = Time.time() # Set up a thread that joins self.taskq and only returns once # all the tasks have completed. The while loops continues as # long as taskq_thread is alive. taskq_thread = Thread(name="taskq", target=self._taskq_worker, args=(self.taskq,)) taskq_thread.start() while taskq_thread.is_alive(): # Wait for a done task to arrive. task = self.doneq().get() # Update the master tasks dict. task_name = task["task_name"] tasks[task_name].update(task) if tasks_pth and save_time.delta() > 10.: # Save task to disk. update_tasks_file(tasks_pth, tasks, overwrite=True) save_time = Time.time() # Report progress. n_done += 1 percent = float(n_done) / n_tasks * 100. dt = t0.delta() time_per_task = dt / float(n_done - n_done0) n_left = n_tasks - n_done t_left = Time(time_per_task * n_left) util.info("[Server] Task `{}` complete:\n\t\t {}/{} ({:.2f}%) {} " "\n\t\t Time left: {}.".format( task_name, n_done, n_tasks, percent, dt, t_left)) # Save tasks to disk one last time. update_tasks_file(tasks_pth, tasks, overwrite=True) # Wait for the clients to d/c. clients = [] while not self.activeq().empty(): clients.append(self.activeq().get()) util.debug("[Server] Waiting for clients to disconnect:\n\t{}.".format( "\n\t".join(clients))) self.activeq().join() time.sleep(0.5) self.activeq().close() self.taskq().close() self.doneq().close() util.info("[Server] Tasks completed.")
def body(self): taskqueue = self.taskqueue outqueue = self.outqueue put = self.put pool = self.pool for taskseq, set_length in iter(taskqueue.get, None): i = -1 for i, task in enumerate(taskseq): if self._state: debug('task handler found thread._state != RUN') break try: put(task) except IOError: debug('could not put task on queue') break else: if set_length: debug('doing set_length()') set_length(i + 1) continue break else: debug('task handler got sentinel') try: # tell result handler to finish when cache is empty debug('task handler sending sentinel to result handler') outqueue.put(None) # tell workers there is no more work debug('task handler sending sentinel to workers') for p in pool: put(None) except IOError: debug('task handler got IOError when sending sentinels') debug('task handler exiting')
break job, i, func, args, kwds = task put((ACK, (job, i, time.time(), pid))) try: result = (True, func(*args, **kwds)) except Exception, e: result = (False, e) try: put((READY, (job, i, result))) except Exception, exc: wrapped = MaybeEncodingError(exc, result[1]) put((READY, (job, i, (False, wrapped)))) completed += 1 debug('worker exiting after %d tasks' % completed) # # Class representing a process pool # class PoolThread(threading.Thread): def __init__(self, *args, **kwargs): threading.Thread.__init__(self) self._state = RUN self.daemon = True def run(self): try:
def timing_handle_tasks(taskqueue, put, outqueue, progressqueue, pool, maxnqueued): thread = threading.current_thread() if progressqueue is not None and hasattr(progressqueue, '_writer'): progressqueue._writer.close() nqueued = 0 for taskseq, set_length in iter(taskqueue.get, None): i = -1 #print 'handle_tasks: task sequence', taskseq for i, task in enumerate(taskseq): # print 'handle_tasks: got task', i if thread._state: debug('task handler found thread._state != RUN') break # print 'N queue:', nqueued, 'max', maxnqueued try: # print 'Queueing new task' put(task) nqueued += 1 except IOError: debug('could not put task on queue') break # print 'N queue:', nqueued, 'max', maxnqueued if progressqueue is not None: while maxnqueued and nqueued >= maxnqueued: try: (job, i, pid) = progressqueue.get() # print 'Job', job, 'element', i, 'pid', pid, 'started' nqueued -= 1 except (IOError, EOFError): break else: if set_length: debug('doing set_length()') set_length(i + 1) continue break else: debug('task handler got sentinel') #print 'debug_handle_tasks got sentinel' try: # tell result handler to finish when cache is empty debug('task handler sending sentinel to result handler') outqueue.put(None) # tell workers there is no more work debug('task handler sending sentinel to workers') for p in pool: put(None) except IOError: debug('task handler got IOError when sending sentinels') # Empty the progressqueue to prevent blocking writing workers? if progressqueue is not None: # print 'task thread: emptying progressqueue' try: # print 'task thread: reading from progressqueue. nqueued=', nqueued (job, i, pid) = progressqueue.get() # print 'Job', job, 'element', i, 'pid', pid, 'started' nqueued -= 1 except (IOError, EOFError): pass
def terminate(self): debug('terminating pool') self._state = TERMINATE self._worker_handler._state = TERMINATE self._terminate()
def set_loky_pickler(loky_pickler=None): global _LokyPickler, _loky_pickler_name if loky_pickler is None: loky_pickler = ENV_LOKY_PICKLER loky_pickler_cls = None # The default loky_pickler is cloudpickle if loky_pickler in ["", None]: loky_pickler = "cloudpickle" if loky_pickler == _loky_pickler_name: return if loky_pickler == "cloudpickle": from joblib.externals.cloudpickle import CloudPickler as loky_pickler_cls else: try: from importlib import import_module module_pickle = import_module(loky_pickler) loky_pickler_cls = module_pickle.Pickler except (ImportError, AttributeError) as e: extra_info = ( "\nThis error occurred while setting loky_pickler to" " '{}', as required by the env variable LOKY_PICKLER" " or the function set_loky_pickler.".format(loky_pickler)) e.args = (e.args[0] + extra_info, ) + e.args[1:] e.msg = e.args[0] raise e util.debug("Using '{}' for serialization.".format( loky_pickler if loky_pickler else "cloudpickle")) class CustomizablePickler(loky_pickler_cls): _loky_pickler_cls = loky_pickler_cls if sys.version_info < (3, ): # Make the dispatch registry an instance level attribute instead of # a reference to the class dictionary under Python 2 _dispatch = loky_pickler_cls.dispatch.copy() _dispatch.update(_ReducerRegistry.dispatch_table) else: # Under Python 3 initialize the dispatch table with a copy of the # default registry _dispatch_table = copyreg.dispatch_table.copy() _dispatch_table.update(_ReducerRegistry.dispatch_table) def __init__(self, writer, reducers=None, protocol=HIGHEST_PROTOCOL): loky_pickler_cls.__init__(self, writer, protocol=protocol) if reducers is None: reducers = {} if sys.version_info < (3, ): self.dispatch = self._dispatch.copy() else: self.dispatch_table = self._dispatch_table.copy() for type, reduce_func in reducers.items(): self.register(type, reduce_func) def register(self, type, reduce_func): """Attach a reducer function to a given type in the dispatch table. """ if sys.version_info < (3, ): # Python 2 pickler dispatching is not explicitly customizable. # Let us use a closure to workaround this limitation. def dispatcher(self, obj): reduced = reduce_func(obj) self.save_reduce(obj=obj, *reduced) self.dispatch[type] = dispatcher else: self.dispatch_table[type] = reduce_func _LokyPickler = CustomizablePickler _loky_pickler_name = loky_pickler
def close(self): debug('closing pool') if self._state == RUN: self._state = CLOSE self._taskqueue.put(None)
def _handle_results(outqueue, get, cache): thread = threading.current_thread() while 1: try: task = get() except (IOError, EOFError): debug('result handler got EOFError/IOError -- exiting') return if thread._state: assert thread._state == TERMINATE debug('result handler found thread._state=TERMINATE') break if task is None: debug('result handler got sentinel') break job, i, obj = task try: cache[job]._set(i, obj) except KeyError: pass while cache and thread._state != TERMINATE: try: task = get() except (IOError, EOFError): debug('result handler got EOFError/IOError -- exiting') return if task is None: debug('result handler ignoring extra sentinel') continue job, i, obj = task try: cache[job]._set(i, obj) except KeyError: pass if hasattr(outqueue, '_reader'): debug('ensuring that outqueue is not full') # If we don't make room available in outqueue then # attempts to add the sentinel (None) to outqueue may # block. There is guaranteed to be no more than 2 sentinels. try: for i in range(10): if not outqueue._reader.poll(): break get() except (IOError, EOFError): pass debug('result handler exiting: len(cache)=%s, thread._state=%s', len(cache), thread._state)
def add_maybe_unlink_finalizer(memmap): util.debug( "[FINALIZER ADD] adding finalizer to {} (id {}, filename {}, pid {})" "".format(type(memmap), id(memmap), os.path.basename(memmap.filename), os.getpid())) weakref.finalize(memmap, _log_and_unlink, memmap.filename)
except AttributeError: if methodname is None: msg = ('#TRACEBACK', format_exc()) else: try: fallback_func = self.fallback_mapping[methodname] result = fallback_func( self, conn, ident, obj, *args, **kwds ) msg = ('#RETURN', result) except Exception: msg = ('#TRACEBACK', format_exc()) except EOFError: util.debug('got EOF -- exiting thread serving %r', threading.current_thread().name) sys.exit(0) except Exception: msg = ('#TRACEBACK', format_exc()) try: try: send(msg) except Exception, e: send(('#UNSERIALIZABLE', repr(msg))) except Exception, e: util.info('exception in thread serving %r', threading.current_thread().name) util.info(' ... message was %r', msg) util.info(' ... exception was %r', e)
def _log_and_unlink(filename): from .externals.loky.backend.resource_tracker import _resource_tracker util.debug("[FINALIZER CALL] object mapping to {} about to be deleted," " decrementing the refcount of the file (pid: {})".format( os.path.basename(filename), os.getpid())) _resource_tracker.maybe_unlink(filename, "file")
def __call__(self, a): m = _get_backing_memmap(a) if m is not None and isinstance(m, np.memmap): # a is already backed by a memmap file, let's reuse it directly return _reduce_memmap_backed(a, m) if (not a.dtype.hasobject and self._max_nbytes is not None and a.nbytes > self._max_nbytes): # check that the folder exists (lazily create the pool temp folder # if required) try: os.makedirs(self._temp_folder) os.chmod(self._temp_folder, FOLDER_PERMISSIONS) except OSError as e: if e.errno != errno.EEXIST: raise e try: basename = self._memmaped_arrays.get(a) except KeyError: # Generate a new unique random filename. The process and thread # ids are only useful for debugging purpose and to make it # easier to cleanup orphaned files in case of hard process # kill (e.g. by "kill -9" or segfault). basename = "{}-{}-{}.pkl".format( os.getpid(), id(threading.current_thread()), uuid4().hex) self._memmaped_arrays.set(a, basename) filename = os.path.join(self._temp_folder, basename) # In case the same array with the same content is passed several # times to the pool subprocess children, serialize it only once is_new_memmap = filename not in self._temporary_memmaped_filenames # add the memmap to the list of temporary memmaps created by joblib self._temporary_memmaped_filenames.add(filename) if self._unlink_on_gc_collect: # Bump reference count of the memmap by 1 to account for # shared usage of the memmap by a child process. The # corresponding decref call will be executed upon calling # resource_tracker.maybe_unlink, registered as a finalizer in # the child. # the incref/decref calls here are only possible when the child # and the parent share the same resource_tracker. It is not the # case for the multiprocessing backend, but it does not matter # because unlinking a memmap from a child process is only # useful to control the memory usage of long-lasting child # processes, while the multiprocessing-based pools terminate # their workers at the end of a map() call. resource_tracker.register(filename, "file") if is_new_memmap: # Incref each temporary memmap created by joblib one extra # time. This means that these memmaps will only be deleted # once an extra maybe_unlink() is called, which is done once # all the jobs have completed (or been canceled) in the # Parallel._terminate_backend() method. resource_tracker.register(filename, "file") if not os.path.exists(filename): util.debug( "[ARRAY DUMP] Pickling new array (shape={}, dtype={}) " "creating a new memmap at {}".format( a.shape, a.dtype, filename)) for dumped_filename in dump(a, filename): os.chmod(dumped_filename, FILE_PERMISSIONS) if self._prewarm: # Warm up the area_data by accessing it. This operation ensures # that the disk access required to create the memmapping # file are performed in the reducing process and avoids # concurrent memmap creation in multiple children # processes. load(filename, mmap_mode=self._mmap_mode).max() else: util.debug( "[ARRAY DUMP] Pickling known array (shape={}, dtype={}) " "reusing memmap file: {}".format( a.shape, a.dtype, os.path.basename(filename))) # The worker process will use joblib.load to memmap the area_data return ((load_temporary_memmap, (filename, self._mmap_mode, self._unlink_on_gc_collect))) else: # do not convert a into memmap, let pickler do its usual copy with # the default system pickler util.debug( '[ARRAY DUMP] Pickling array (NO MEMMAPPING) (shape={}, ' ' dtype={}).'.format(a.shape, a.dtype)) return (loads, (dumps(a, protocol=HIGHEST_PROTOCOL), ))
def on_state_change(task): state, args = task try: state_handlers[state](*args) except KeyError: debug("Unknown job state: %s (args=%s)" % (state, args))
debug('worker got sentinel -- exiting') break job, i, func, args, kwds = task try: result = (True, func(*args, **kwds)) except Exception, e: result = (False, e) try: put((job, i, result)) except Exception as e: wrapped = create_detailed_pickling_error(e, result[1]) put((job, i, (False, wrapped))) completed += 1 debug('worker exiting after %d tasks' % completed) def create_detailed_pickling_error(exception, instance): """ MaybeEncodingError - PicklingError: Can't pickle dictproxy #8748 :param instance: The instance we failed to encode :return: We return the MaybeEncodingError, we include lots of information that allow me to debug what's going wrong. """ attribute = None def can_pickle(data): try: cPickle.dumps(v)
def terminate(self): debug('terminating pool') self._state = TERMINATE self._terminate()
def _handle_tasks(taskqueue, put, outqueue, pool, cache): thread = threading.current_thread() for taskseq, set_length in iter(taskqueue.get, None): i = -1 for i, task in enumerate(taskseq): if thread._state: debug('task handler found thread._state != RUN') break try: put(task) except Exception as e: job, ind = task[:2] try: cache[job]._set(ind, (False, e)) except KeyError: pass else: if set_length: debug('doing set_length()') set_length(i + 1) continue break else: debug('task handler got sentinel') try: # tell result handler to finish when cache is empty debug('task handler sending sentinel to result handler') outqueue.put(None) # tell workers there is no more work debug('task handler sending sentinel to workers') for p in pool: put(None) except IOError: debug('task handler got IOError when sending sentinels') debug('task handler exiting')
class ResultHandler(PoolThread): def __init__(self, outqueue, get, cache, poll, join_exited_workers, putlock): self.outqueue = outqueue self.get = get self.cache = cache self.poll = poll self.join_exited_workers = join_exited_workers self.putlock = putlock super(ResultHandler, self).__init__() def body(self): get = self.get outqueue = self.outqueue cache = self.cache poll = self.poll join_exited_workers = self.join_exited_workers putlock = self.putlock def on_ack(job, i, time_accepted, pid): try: cache[job]._ack(i, time_accepted, pid) except (KeyError, AttributeError): # Object gone or doesn't support _ack (e.g. IMAPIterator). pass def on_ready(job, i, obj): try: item = cache[job] except KeyError: return if not item.ready(): if putlock is not None: putlock.release() try: item._set(i, obj) except KeyError: pass state_handlers = {ACK: on_ack, READY: on_ready} def on_state_change(task): state, args = task try: state_handlers[state](*args) except KeyError: debug("Unknown job state: %s (args=%s)" % (state, args)) debug('result handler starting') while 1: try: ready, task = poll(0.2) except (IOError, EOFError), exc: debug('result handler got %r -- exiting' % (exc, )) return if self._state: assert self._state == TERMINATE debug('result handler found thread._state=TERMINATE') break if ready: if task is None: debug('result handler got sentinel') break on_state_change(task) time_terminate = None while cache and self._state != TERMINATE: try: ready, task = poll(0.2) except (IOError, EOFError), exc: debug('result handler got %r -- exiting' % (exc, )) return if ready: if task is None: debug('result handler ignoring extra sentinel') continue on_state_change(task) try: join_exited_workers(shutdown=True) except WorkersJoined: now = time.time() if not time_terminate: time_terminate = now else: if now - time_terminate > 5.0: debug('result handler exiting: timed out') break debug('result handler: all workers terminated, ' 'timeout in %ss' % (abs(min(now - time_terminate - 5.0, 0))))
def close(self): debug('closing pool') if self._state == RUN: self._state = CLOSE self._worker_handler._state = CLOSE
class TimeoutHandler(PoolThread): def __init__(self, processes, cache, t_soft, t_hard): self.processes = processes self.cache = cache self.t_soft = t_soft self.t_hard = t_hard super(TimeoutHandler, self).__init__() def body(self): processes = self.processes cache = self.cache t_hard, t_soft = self.t_hard, self.t_soft dirty = set() def _process_by_pid(pid): for index, process in enumerate(processes): if process.pid == pid: return process, index return None, None def _timed_out(start, timeout): if not start or not timeout: return False if time.time() >= start + timeout: return True def _on_soft_timeout(job, i, soft_timeout): debug('soft time limit exceeded for %i' % i) process, _index = _process_by_pid(job._worker_pid) if not process: return # Run timeout callback if job._timeout_callback is not None: job._timeout_callback(soft=True, timeout=soft_timeout) try: os.kill(job._worker_pid, SIG_SOFT_TIMEOUT) except OSError, exc: if exc.errno == errno.ESRCH: pass else: raise dirty.add(i) def _on_hard_timeout(job, i, hard_timeout): if job.ready(): return debug('hard time limit exceeded for %i', i) # Remove from cache and set return value to an exception job._set(i, (False, TimeLimitExceeded(hard_timeout))) # Remove from _pool process, _index = _process_by_pid(job._worker_pid) # Run timeout callback if job._timeout_callback is not None: job._timeout_callback(soft=False, timeout=hard_timeout) if process: process.terminate() # Inner-loop while self._state == RUN: # Remove dirty items not in cache anymore if dirty: dirty = set(k for k in dirty if k in cache) for i, job in cache.items(): ack_time = job._time_accepted soft_timeout = job._soft_timeout if soft_timeout is None: soft_timeout = t_soft hard_timeout = job._timeout if hard_timeout is None: hard_timeout = t_hard if _timed_out(ack_time, hard_timeout): _on_hard_timeout(job, i, hard_timeout) elif i not in dirty and _timed_out(ack_time, soft_timeout): _on_soft_timeout(job, i, soft_timeout) time.sleep(0.5) # Don't waste CPU cycles. debug('timeout handler exiting')
def body(self): debug('worker handler starting') while self._state == RUN and self.pool._state == RUN: self.pool._maintain_pool() time.sleep(0.8) debug('worker handler exiting')
def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, worker_handler, task_handler, result_handler, cache): # this is guaranteed to only be called once debug('finalizing pool') worker_handler._state = TERMINATE task_handler._state = TERMINATE debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) assert result_handler.is_alive() or len(cache) == 0 result_handler._state = TERMINATE outqueue.put(None) # sentinel # We must wait for the worker handler to exit before terminating # workers because we don't want workers to be restarted behind our back. debug('joining worker handler') if threading.current_thread() is not worker_handler: worker_handler.join(1e100) # Terminate workers which haven't already finished. if pool and hasattr(pool[0], 'terminate'): debug('terminating workers') for p in pool: if p.exitcode is None: try: p.terminate() except AttributeError: # https://github.com/andresriancho/w3af/issues/9361 continue debug('joining task handler') if threading.current_thread() is not task_handler: task_handler.join(1e100) debug('joining result handler') if threading.current_thread() is not result_handler: result_handler.join(1e100) if pool and hasattr(pool[0], 'terminate'): debug('joining pool workers') for p in pool: if p.is_alive(): # worker has not yet exited debug('cleaning up worker %d' % p.pid) p.join()
def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, worker_handler, task_handler, result_handler, cache): # this is guaranteed to only be called once debug('finalizing pool') worker_handler._state = TERMINATE task_handler._state = TERMINATE debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) assert result_handler.is_alive() or len(cache) == 0 result_handler._state = TERMINATE outqueue.put(None) # sentinel # We must wait for the worker handler to exit before terminating # workers because we don't want workers to be restarted behind our back. debug('joining worker handler') worker_handler.join() # Terminate workers which haven't already finished. if pool and hasattr(pool[0], 'terminate'): debug('terminating workers') for p in pool: if p.exitcode is None: p.terminate() debug('joining task handler') task_handler.join(1e100) debug('joining result handler') result_handler.join(1e100) if pool and hasattr(pool[0], 'terminate'): debug('joining pool workers') for p in pool: if p.is_alive(): # worker has not yet exited debug('cleaning up worker %d' % p.pid) p.join()
def join_thread(self): debug('Queue.join_thread()') assert self._closed if self._jointhread: self._jointhread()
def __setstate__(self, state): self._semlock = _SemLock._rebuild(*state) util.debug( f'recreated blocker with handle {state[0]!r} and name "{state[3]}"' ) self._make_methods()
def __setstate__(self, state): self._semlock = _SemLock._rebuild(*state) util.debug('recreated blocker with handle %r and name "%s"' % (state[0], state[3])) self._make_methods()
def main(fd, verbose=0): '''Run resource tracker.''' # protect the process from ^C and "killall python" etc if verbose: util.log_to_stderr(level=util.DEBUG) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass if verbose: util.debug("Main resource tracker is running") registry = {rtype: dict() for rtype in _CLEANUP_FUNCS.keys()} try: # keep track of registered/unregistered resources if sys.platform == "win32": fd = msvcrt.open_osfhandle(fd, os.O_RDONLY) with open(fd, 'rb') as f: while True: line = f.readline() if line == b'': # EOF break try: splitted = line.strip().decode('ascii').split(':') # name can potentially contain separator symbols (for # instance folders on Windows) cmd, name, rtype = (splitted[0], ':'.join(splitted[1:-1]), splitted[-1]) if cmd == 'PROBE': continue if rtype not in _CLEANUP_FUNCS: raise ValueError( 'Cannot register {} for automatic cleanup: ' 'unknown resource type ({}). Resource type should ' 'be one of the following: {}'.format( name, rtype, list(_CLEANUP_FUNCS.keys()))) if cmd == 'REGISTER': if name not in registry[rtype]: registry[rtype][name] = 1 else: registry[rtype][name] += 1 if verbose: util.debug( "[ResourceTracker] incremented refcount of {} " "{} (current {})".format( rtype, name, registry[rtype][name])) elif cmd == 'UNREGISTER': del registry[rtype][name] if verbose: util.debug("[ResourceTracker] unregister {} {}: " "registry({})".format( name, rtype, len(registry))) elif cmd == 'MAYBE_UNLINK': registry[rtype][name] -= 1 if verbose: util.debug( "[ResourceTracker] decremented refcount of {} " "{} (current {})".format( rtype, name, registry[rtype][name])) if registry[rtype][name] == 0: del registry[rtype][name] try: if verbose: util.debug( "[ResourceTracker] unlink {}".format( name)) _CLEANUP_FUNCS[rtype](name) except Exception as e: warnings.warn('resource_tracker: %s: %r' % (name, e)) else: raise RuntimeError('unrecognized command %r' % cmd) except BaseException: try: sys.excepthook(*sys.exc_info()) except BaseException: pass finally: # all processes have terminated; cleanup any remaining resources def _unlink_resources(rtype_registry, rtype): if rtype_registry: try: warnings.warn('resource_tracker: There appear to be %d ' 'leaked %s objects to clean up at shutdown' % (len(rtype_registry), rtype)) except Exception: pass for name in rtype_registry: # For some reason the process which created and registered this # resource has failed to unregister it. Presumably it has # died. We therefore clean it up. try: _CLEANUP_FUNCS[rtype](name) if verbose: util.debug("[ResourceTracker] unlink {}".format(name)) except Exception as e: warnings.warn('resource_tracker: %s: %r' % (name, e)) for rtype, rtype_registry in registry.items(): if rtype == "folder": continue else: _unlink_resources(rtype_registry, rtype) # The default cleanup routine for folders deletes everything inside # those folders recursively, which can include other resources tracked # by the resource tracker). To limit the risk of the resource tracker # attempting to delete twice a resource (once as part of a tracked # folder, and once as a resource), we delete the folders after all # other resource types. if "folder" in registry: _unlink_resources(registry["folder"], "folder") if verbose: util.debug("resource tracker shut down")
def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, worker_handler, task_handler, result_handler, cache, timeout_handler): # this is guaranteed to only be called once debug('finalizing pool') worker_handler.terminate() task_handler.terminate() taskqueue.put(None) # sentinel debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) result_handler.terminate() outqueue.put(None) # sentinel if timeout_handler is not None: timeout_handler.terminate() # Terminate workers which haven't already finished if pool and hasattr(pool[0], 'terminate'): debug('terminating workers') for p in pool: if p.exitcode is None: p.terminate() debug('joining task handler') task_handler.join(1e100) debug('joining result handler') result_handler.join(1e100) if timeout_handler is not None: debug('joining timeout handler') timeout_handler.join(1e100) if pool and hasattr(pool[0], 'terminate'): debug('joining pool workers') for p in pool: if p.is_alive(): # worker has not yet exited debug('cleaning up worker %d' % p.pid) p.join() debug('pool workers joined')
def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, task_handler, result_handler, cache): # this is guaranteed to only be called once debug('finalizing pool') task_handler._state = TERMINATE taskqueue.put(None) # sentinel debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) assert result_handler.is_alive() or len(cache) == 0 result_handler._state = TERMINATE outqueue.put(None) # sentinel if pool and hasattr(pool[0], 'terminate'): debug('terminating workers') for p in pool: p.terminate() debug('joining task handler') task_handler.join(1e100) debug('joining result handler') result_handler.join(1e100) if pool and hasattr(pool[0], 'terminate'): debug('joining pool workers') for p in pool: p.join()
def __setstate__(self, state): self._semlock = multiprocessing.SemLock._rebuild(*state) debug('recreated blocker with handle %r' % state[0]) self._make_methods()
def serve_client(self, conn): """ Handle requests from the proxies in a particular process/thread """ util.debug('starting server thread to service %r', threading.current_thread().name) recv = conn.recv send = conn.send id_to_obj = self.id_to_obj while not self.stop: try: methodname = obj = None request = recv() ident, methodname, args, kwds = request obj, exposed, gettypeid = id_to_obj[ident] if methodname not in exposed: raise AttributeError( 'method %r of %r object is not in exposed=%r' % (methodname, type(obj), exposed)) function = getattr(obj, methodname) try: res = function(*args, **kwds) except Exception as e: msg = ('#ERROR', e) else: typeid = gettypeid and gettypeid.get(methodname, None) if typeid: rident, rexposed = self.create(conn, typeid, res) token = Token(typeid, self.address, rident) msg = ('#PROXY', (rexposed, token)) else: msg = ('#RETURN', res) except AttributeError: if methodname is None: msg = ('#TRACEBACK', format_exc()) else: try: fallback_func = self.fallback_mapping[methodname] result = fallback_func(self, conn, ident, obj, *args, **kwds) msg = ('#RETURN', result) except Exception: msg = ('#TRACEBACK', format_exc()) except EOFError: util.debug('got EOF -- exiting thread serving %r', threading.current_thread().name) sys.exit(0) except Exception: msg = ('#TRACEBACK', format_exc()) try: try: send(msg) except Exception as e: send(('#UNSERIALIZABLE', repr(msg))) except Exception as e: util.info('exception in thread serving %r', threading.current_thread().name) util.info(' ... message was %r', msg) util.info(' ... exception was %r', e) conn.close() sys.exit(1) return
def ensure_running(self): '''Make sure that resource tracker process is running. This can be run from any process. Usually a child process will use the resource created by its parent.''' with self._lock: if self._fd is not None: # resource tracker was launched before, is it still running? if self._check_alive(): # => still alive return # => dead, launch it again os.close(self._fd) if os.name == "posix": try: # At this point, the resource_tracker process has been # killed or crashed. Let's remove the process entry # from the process table to avoid zombie processes. os.waitpid(self._pid, 0) except OSError: # The process was terminated or is a child from an # ancestor of the current process. pass self._fd = None self._pid = None warnings.warn('resource_tracker: process died unexpectedly, ' 'relaunching. Some folders/sempahores might ' 'leak.') fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: pass r, w = os.pipe() if sys.platform == "win32": _r = duplicate(msvcrt.get_osfhandle(r), inheritable=True) os.close(r) r = _r cmd = 'from {} import main; main({}, {})'.format( main.__module__, r, VERBOSE) try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() # In python 3.3, there is a bug which put `-RRRRR..` instead of # `-R` in args. Replace it to get the correct flags. # See https://github.com/python/cpython/blob/3.3/Lib/subprocess.py#L488 if sys.version_info[:2] <= (3, 3): import re for i in range(1, len(args)): args[i] = re.sub("-R+", "-R", args[i]) args += ['-c', cmd] util.debug("launching resource tracker: {}".format(args)) # bpo-33613: Register a signal mask that will block the # signals. This signal mask will be inherited by the child # that is going to be spawned and will protect the child from a # race condition that can make the child die before it # registers signal handlers for SIGINT and SIGTERM. The mask is # unregistered after spawning the child. try: if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) pid = spawnv_passfds(exe, args, fds_to_pass) finally: if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) except BaseException: os.close(w) raise else: self._fd = w self._pid = pid finally: if sys.platform == "win32": _winapi.CloseHandle(r) else: os.close(r)