def _bootstrap(self): from multiprocessing import util global _current_process try: self._children = set() self._counter = itertools.count(1) try: # sys.stdin.close() sys.stdin = open(os.devnull) except (OSError, ValueError): pass _current_process = self util._finalizer_registry.clear() util._run_after_forkers() util.info('child process calling self.run()') try: self.run() exitcode = 0 finally: pass # util._exit_function() except SystemExit, e: if not e.args: exitcode = 1 elif isinstance(e.args[0], int): exitcode = e.args[0] else: sys.stderr.write(str(e.args[0]) + '\n') sys.stderr.flush() exitcode = 1
def set_shared(self, params, tasks, force=False): """Configures the mandatory shared resources. Updates `params`. Places tasks in `taskq`. Args: params (dict): Parameters. tasks (dict): Tasks. Keyword Args: force (bool): Forces task into `taskq`, regardless of whether it is already complete. """ # Create a client that will the set the actual resource data. params["save"] = self.save client = SCManager(user=self.user, address=self.address, authkey=self.authkey) client.connect() # Update params. client.params().update(params) # Update tasks. taskq = client.taskq() for task in tasks.itervalues(): if force or not task.get("complete", False): taskq.put(task) util.info("[Server] Shared data successfully created.") del client
def _send_result(local, remote, retries): """Send a task's result file to remote location. Args: local (str): Path of local file. remote (str): Path of remote file. retries (int): Number of retries to perform. Return: (bool): Indicates whether send was successful. """ send_str = "{} (local) > {} (remote)".format(local, remote) sent_pths = () for r in xrange(retries): sent_pths = ssh.put(local, remote) if sent_pths.succeeded and remote in sent_pths: sent = True break else: msg = "[Client] {}/{} failed send retries ({}){{}}".format( r + 1, retries, send_str) if r + 1 < retries: util.info(msg.format(", retrying...")) else: util.info(msg.format(", exiting.")) else: sent = False # sys.exit(0) # raise NetworkError(msg.format(".")) return sent
def bootstrap_2_6_6(self): """Pulled from python 2.6.6. Needed to ensure we have the fix from http://bugs.python.org/issue5313 when running on python version 2.6.2 or lower.""" try: self._children = set() self._counter = itertools.count(1) try: sys.stdin.close() sys.stdin = open(os.devnull) except (OSError, ValueError): pass multiprocessing._current_process = self util._finalizer_registry.clear() util._run_after_forkers() util.info("child process calling self.run()") try: self.run() exitcode = 0 finally: util._exit_function() except SystemExit, e: if not e.args: exitcode = 1 elif type(e.args[0]) is int: exitcode = e.args[0] else: sys.stderr.write(e.args[0] + "\n") sys.stderr.flush() exitcode = 1
def _after_fork(self): self._manager = None try: self._incref() except Exception, e: # the proxy may just be for a manager which has shutdown util.info('incref failed: %s' % e)
def _bootstrap(self): from multiprocessing import util global _current_process try: self._children = set() self._counter = itertools.count(1) try: sys.stdin.close() sys.stdin = open(os.devnull) except (OSError, ValueError): pass _current_process = self util._finalizer_registry.clear() util._run_after_forkers() util.info('child process calling self.run()') try: self.run() exitcode = 0 finally: util._exit_function() except SystemExit, e: if not e.args: exitcode = 1 elif type(e.args[0]) is int: exitcode = e.args[0] else: sys.stderr.write(e.args[0] + '\n') sys.stderr.flush() exitcode = 1
def shutdown(self, c): try: try: util.debug('manager received shutdown message') c.send(('#RETURN', None)) if sys.stdout != sys.__stdout__: util.debug('resetting stdout, stderr') sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ util._run_finalizers(0) for p in active_children(): util.debug('terminating a child process of manager') p.terminate() for p in active_children(): util.debug('terminating a child process of manager') p.join() util._run_finalizers() util.info('manager exiting with exitcode 0') except: import traceback traceback.print_exc() finally: exit(0) return
def shutdown(self, c): ''' Shutdown this process ''' try: try: util.debug('manager received shutdown message') c.send(('#RETURN', None)) if sys.stdout != sys.__stdout__: util.debug('resetting stdout, stderr') sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ util._run_finalizers(0) for p in active_children(): util.debug('terminating a child process of manager') p.terminate() for p in active_children(): util.debug('terminating a child process of manager') p.join() util._run_finalizers() util.info('manager exiting with exitcode 0') except: import traceback traceback.print_exc() finally: exit(0)
def _after_fork(self): self._manager = None try: self._incref() except Exception as e: # the proxy may just be for a manager which has shutdown util.info('incref failed: %s' % e)
def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe, queue_sem): while True: try: with notempty: if not buffer: notempty.wait() try: while True: obj = buffer.popleft() if obj is _sentinel: debug('feeder thread got sentinel -- exiting') close() return obj = ForkingPickler.dumps(obj) if sys.platform == 'win32': send_bytes(obj) else: with writelock: send_bytes(obj) except IndexError: # 当buffer为空时popleft会抛出异常 pass except Exception as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: return # Since this runs in a daemon thread the resources it uses may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has started to cleanup. if is_exiting(): info('error in queue thread: %s', e) return else: # Since the object has not been sent in the queue, we need to decrease the size of the queue. # The error acts as if the object had been silently removed from the queue and this step is necessary to have a properly working queue. queue_sem.release() traceback.print_exc()
def _feed(buffer, notempty, send, writelock, close, ignore_epipe): debug('starting thread to feed data to pipe') from .util import is_exiting nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != 'win32': wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None try: while 1: nacquire() try: if not buffer: nwait() finally: nrelease() try: while 1: obj = bpopleft() if obj is sentinel: debug('feeder thread got sentinel -- exiting') close() return if wacquire is None: send(obj) # Delete references to object. See issue16284 del obj else: wacquire() try: send(obj) # Delete references to object. See issue16284 del obj finally: wrelease() except IndexError: pass except Exception as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. try: if is_exiting(): info('error in queue thread: %s', e) else: import traceback traceback.print_exc() except Exception: pass
def _after_fork(self): self._manager = None try: self._incref() except Exception as e: util.info('incref failed: %s' % e) return
def _run_server(cls, registry, address, authkey, serializer, writer, initializer = None, initargs = ()): if initializer is not None: initializer(*initargs) server = cls._Server(registry, address, authkey, serializer) writer.send(server.address) writer.close() util.info('manager serving at %r', server.address) server.serve_forever()
def _feed(buffer, notempty, send_bytes, writelock, close, reducers, ignore_epipe, onerror, queue_sem): util.debug('starting thread to feed data to pipe') nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != 'win32': wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None while 1: try: nacquire() try: if not buffer: nwait() finally: nrelease() try: while 1: obj = bpopleft() if obj is sentinel: util.debug('feeder thread got sentinel -- exiting') close() return # serialize the data before acquiring the lock obj_ = CustomizableLokyPickler.dumps( obj, reducers=reducers) if wacquire is None: send_bytes(obj_) else: wacquire() try: send_bytes(obj_) finally: wrelease() # Remove references early to avoid leaking memory del obj, obj_ except IndexError: pass except BaseException as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. if util.is_exiting(): util.info('error in queue thread: %s', e) return else: queue_sem.release() onerror(e, obj)
def _feed(buffer, notempty, send, writelock, close, ignore_epipe): debug('starting thread to feed data to pipe') from .util import is_exiting nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != 'win32': wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None try: while 1: nacquire() try: if not buffer: nwait() finally: nrelease() try: while 1: obj = bpopleft() if obj is sentinel: debug('feeder thread got sentinel -- exiting') close() return if wacquire is None: send(obj) else: wacquire() try: send(obj) finally: wrelease() except IndexError: pass except Exception as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. try: if is_exiting(): info('error in queue thread: %s', e) else: import traceback traceback.print_exc() except Exception: pass
def _run_server(cls, registry, address, authkey, serializer, writer, initializer = None, initargs = ()): """ Create a server, report its address and run it """ if initializer is not None: initializer(*initargs) server = cls._Server(registry, address, authkey, serializer) writer.send(server.address) writer.close() util.info('manager serving at %r', server.address) server.serve_forever() return
def _feed(buffer, notempty, send, writelock, close): debug('starting thread to feed data to pipe') from .util import is_exiting nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != 'win32': wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None try: while 1: nacquire() try: if not buffer: nwait() finally: nrelease() try: while 1: obj = bpopleft() if obj is sentinel: debug('feeder thread got sentinel -- exiting') close() return if wacquire is None: send(obj) else: wacquire() try: send(obj) finally: wrelease() except IndexError: pass except Exception as e: try: if is_exiting(): info('error in queue thread: %s', e) else: import traceback traceback.print_exc() except Exception: pass return
def _finalize_manager(process, address, authkey, state, _Client): ''' Shutdown the manager process; will be registered as a finalizer ''' if process.is_alive(): util.info('sending shutdown message to manager') try: conn = _Client(address, authkey=authkey) try: managers.dispatch(conn, None, 'shutdown') finally: conn.close() except Exception: pass process.join(timeout=0.2) if process.is_alive(): util.info('manager still alive') if hasattr(process, 'terminate'): util.info('trying to `terminate()` manager process') try: process.terminate() process.join(timeout=0.1) # XXX: catch the OS error ... something weird is going on here.. except OSError: pass if process.is_alive(): util.info('manager still alive after terminate') state.value = managers.State.SHUTDOWN try: del managers.BaseProxy._address_to_local[address] except KeyError: pass
def _run_server(cls, registry, address, authkey, serializer, writer, conn_writer, initializer=None, initargs=()): """ Create a server, report its address and run it.""" if initializer is not None: initializer(*initargs) # Create server. server = cls._Server(registry, fqaddr(address), authkey, serializer, conn_writer) # Inform parent process of the server's address. writer.send(server.address) writer.close() # Run the manager. util.info("Server running at {}:{}.".format(*server.address)) server.serve_forever()
def _finalize_manager(process, address, authkey, state, conns, _Client): """ Shutdown the manager process; will be registered as a finalizer.""" if process.is_alive(): util.info("Sending shutdown message to manager.") try: conn = _Client(fqaddr(address), authkey=authkey) try: dispatch(conn, None, "shutdown") finally: conn.close() except Exception: pass for conn in conns: conn.close() process.join(timeout=0.2) if process.is_alive(): util.info("Manager still alive.") if hasattr(process, "terminate"): util.info("Trying to `terminate()` manager process.") process.terminate() process.join(timeout=0.1) if process.is_alive(): util.info("Manager still alive after terminate.") state.value = State.SHUTDOWN try: del BaseProxy._address_to_local[fqaddr(address)] except KeyError: pass
def _finalize_manager(process, address, authkey, state, _Client): if process.is_alive(): util.info('sending shutdown message to manager') try: conn = _Client(address, authkey=authkey) try: dispatch(conn, None, 'shutdown') finally: conn.close() except Exception: pass process.join(timeout=0.2) if process.is_alive(): util.info('manager still alive') if hasattr(process, 'terminate'): util.info('trying to `terminate()` manager process') process.terminate() process.join(timeout=0.1) if process.is_alive(): util.info('manager still alive after terminate') state.value = State.SHUTDOWN try: del BaseProxy._address_to_local[address] except KeyError: pass
def _run_server(cls, registry, address, authkey, serializer, writer): ''' Create a server, report its address and run it ''' # create server server = cls._Server(registry, address, authkey, serializer) # inform parent process of the server's address writer.send(server.address) writer.close() # run the manager util.info('manager serving at %r', server.address) server.serve_forever()
def _finalize_manager(process, address, authkey, state, _Client): if process.is_alive(): util.info('sending shutdown message to manager') try: conn = _Client(address, authkey=authkey) try: dispatch(conn, None, 'shutdown') finally: conn.close() except Exception: pass process.join(timeout=0.2) if process.is_alive(): util.info('manager still alive') if hasattr(process, 'terminate'): util.info('trying to `terminate()` manager process') process.terminate() process.join(timeout=0.1) if process.is_alive(): util.info('manager still alive after terminate') state.value = State.SHUTDOWN try: del BaseProxy._address_to_local[address] except KeyError: pass return
def _worker_process(self, finish, job, taskq, doneq, max_run_retries=3, max_send_retries=10): """Run a set of jobs.""" job.setup() run_retries = 0 exitcode = 0 # Loop over tasks. while not taskq.empty(): # Pop task off queue. task = taskq.get() success = False T0 = Time.time() try: progress, tmp_fid = job.run(task) except EOFError as err: # Report what went wrong and retry. util.debug("[Client] {}".format(err.msg)) msg = "[Client] {}/{} failed run retries, {{}}".format( run_retries, max_run_retries) if run_retries < max_run_retries: run_retries += 1 util.debug(msg.format("retrying...")) else: util.debug(msg.format("exiting.")) raise err else: # Send results. sent = send_result(tmp_fid, task, retries=max_send_retries) if sent or not self.save: # Mark simulation as complete. task["complete"] = True # Task is done. doneq.put(task) success = True # Report progress. progress.task = (taskq.qsize(), T0.delta()) progress.report() finally: if not success: # Task did not complete successfully: put it back in taskq. taskq.put(task) taskq.task_done() exitcode = 0 if finish.is_set(): exitcode = 100 break job.teardown() util.info("[Client] Process complete: {}.".format(ProcLabel())) sys.exit(exitcode)
def _global_after_fork(): # Previously every app would call: # `register_after_fork(app, app._after_fork)` # but this created a leak as `register_after_fork` stores concrete object # references and once registered an object cannot be removed without # touching and iterating over the private afterfork registry list. # # See Issue #1949 from celery import _state from multiprocessing.util import info for app in _state.apps: try: app._after_fork() except Exception as exc: info('after forker raised exception: %r' % (exc, ), exc_info=1)
def _feed(buffer, notempty, send, writelock, close): debug('starting thread to feed data to pipe') from .util import is_exiting nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel wacquire = writelock.acquire wrelease = writelock.release try: while 1: try: while 1: obj = bpopleft() if obj is sentinel: debug('feeder thread got sentinel -- exiting') close() return if wacquire is None: send(obj) else: wacquire() try: send(obj) finally: wrelease() except IndexError: pass yield "Done for now" except Exception as e: # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. try: if is_exiting(): info('error in queue thread: %s', e) else: import traceback traceback.print_exc() except Exception: pass
def _malloc(self, size): i = bisect.bisect_left(self._lengths, size) if i == len(self._lengths): length = self._roundup(max(self._size, size), mmap.PAGESIZE) self._size *= 2 info('allocating a new mmap of length %d', length) arena = Arena(length) self._arenas.append(arena) return (arena, 0, length) length = self._lengths[i] seq = self._len_to_seq[length] block = seq.pop() if not seq: del self._len_to_seq[length] del self._lengths[i] arena, start, stop = block del self._start_to_block[arena, start] del self._stop_to_block[arena, stop] return block
def get_shared(self): """Get shared data from server. Return: (tuple: dict, multiprocessing.JoinableQueue, multiprocessing.Queue, multiprocessing.JoinableQueue): Parameters, shared queue for tasks, shared queue for done tasks, shared queue for active connections. """ params = self.params() self.Job = params["Job"] taskq = self.taskq() doneq = self.doneq() activeq = self.activeq() util.info("[Client] Received shared data.") # Notify the server that this client is active. activeq.put(self.host) return params, taskq, doneq, activeq
def start(self, params, tasks, force=False): """Start server. Args: params (dict): Parameters. tasks (dict): Tasks. Keyword Args: force (bool): Forces task into `taskq`, regardless of whether it is already complete. """ super(ServerManager, self).start() self.set_shared(params, tasks, force=force) # Monitor the tasks and update the tasks file as completed tasks # arrive from the clients. self.manage_tasks(tasks) self.shutdown() util.info("[Server] Jobs complete. Shutting down.") sys.exit(0)
def _malloc(self, size): # returns a large enough block -- it might be much larger i = bisect.bisect_left(self._lengths, size) if i == len(self._lengths): length = self._roundup(max(self._size, size), mmap.PAGESIZE) self._size *= 2 info('allocating a new mmap of length %d', length) arena = Arena(length) self._arenas.append(arena) return (arena, 0, length) else: length = self._lengths[i] seq = self._len_to_seq[length] block = seq.pop() if not seq: del self._len_to_seq[length], self._lengths[i] (arena, start, stop) = block del self._start_to_block[(arena, start)] del self._stop_to_block[(arena, stop)] return block
def _spawn(self, finish, params, taskq, doneq, args, kwargs): """Spawn a job process. Args: finish (multiprocessing.Event): Signals that process is finished. params (dict): Parameters. taskq (multiprocessing.JoinableQueue): Shared queue for tasks. doneq (multiprocessing.Queue): Shared queue for done tasks. args (tuple): Arguments for ``_worker_process``. kwargs (dict): Keyword arguments for ``_worker_process``. Return: (multiprocessing.Process): Worker process. """ # Initialize the job. job = self.Job(params, *args, **kwargs) proc = Process(target=self._worker_process, args=(finish, job, taskq, doneq)) util.info("[Client] Spawning process: '{}'".format(proc.name)) return proc
def handle_request(self, c): funcname = result = request = None try: connection.deliver_challenge(c, self.authkey) connection.answer_challenge(c, self.authkey) request = c.recv() ignore, funcname, args, kwds = request assert funcname in self.public, '%r unrecognized' % funcname func = getattr(self, funcname) except Exception: msg = ('#TRACEBACK', format_exc()) else: try: result = func(c, *args, **kwds) except Exception: msg = ('#TRACEBACK', format_exc()) else: msg = ('#RETURN', result) try: c.send(msg) except Exception as e: try: c.send(('#TRACEBACK', format_exc())) except Exception: pass util.info('Failure to send message: %r', msg) util.info(' ... request was %r', request) util.info(' ... exception was %r', e) c.close()
def report(self): """ Outputs formatted progress text.""" util.info("\n") if self.task: util.info(self.task) if self.msg: util.info(self.msg)
def bootstrap_2_6_6(self): """Pulled from python 2.6.6. Needed to ensure we have the fix from http://bugs.python.org/issue5313 when running on python version 2.6.2 or lower.""" try: self._children = set() self._counter = itertools.count(1) try: sys.stdin.close() sys.stdin = open(os.devnull) except (OSError, ValueError): pass multiprocessing._current_process = self util._finalizer_registry.clear() util._run_after_forkers() util.info('child process calling self.run()') try: self.run() exitcode = 0 finally: util._exit_function() except SystemExit as e: if not e.args: exitcode = 1 elif type(e.args[0]) is int: exitcode = e.args[0] else: sys.stderr.write(e.args[0] + '\n') sys.stderr.flush() exitcode = 1 except: exitcode = 1 import traceback sys.stderr.write('Process %s:\n' % self.name) sys.stderr.flush() traceback.print_exc() util.info('process exiting with exitcode %d' % exitcode) return exitcode
def handle_request(self, c): funcname = result = request = None try: connection.deliver_challenge(c, self.authkey) connection.answer_challenge(c, self.authkey) request = c.recv() ignore, funcname, args, kwds = request func = getattr(self, funcname) except Exception: msg = ('#TRACEBACK', format_exc()) else: try: result = func(c, *args, **kwds) except Exception: msg = ('#TRACEBACK', format_exc()) else: msg = ('#RETURN', result) try: c.send(msg) except Exception as e: try: c.send(('#TRACEBACK', format_exc())) except Exception: pass util.info('Failure to send message: %r', msg) util.info(' ... request was %r', request) util.info(' ... exception was %r', e) c.close() return
def handle_request(self, c): ''' Handle a new connection ''' funcname = result = request = None try: connection.deliver_challenge(c, self.authkey) connection.answer_challenge(c, self.authkey) request = c.recv() ignore, funcname, args, kwds = request assert funcname in self.public, '%r unrecognized' % funcname func = getattr(self, funcname) except Exception: msg = ('#TRACEBACK', format_exc()) else: try: result = func(c, *args, **kwds) except Exception: msg = ('#TRACEBACK', format_exc()) else: msg = ('#RETURN', result) try: c.send(msg) except Exception, e: try: c.send(('#TRACEBACK', format_exc())) except Exception: pass util.info('Failure to send message: %r', msg) util.info(' ... request was %r', request) util.info(' ... exception was %r', e)
def start(self, n_procs=None, args=(), kwargs={}, timeout=1e5): """ Start the client. Keyword Args: n_procs (int): Number of processes to run. args (tuple): Arguments for ``_worker_process``. kwargs (dict): Keyword arguments for ``_worker_process``. timeout (float): Time to run process before signaling that it be restarted. This circumvents memory leaks. """ if n_procs is None: n_procs = cpu_count() self.connect() params, taskq, doneq, activeq = self.get_shared() kwargs["save"] = params.pop("save", False) if kwargs["save"]: setup_send(self.user, self.address[0]) # Arguments for each thread's worker. worker_args = (params, taskq, doneq, args, kwargs) worker_kwargs = dict(timeout=timeout) threads = [] # Control each process with a Thread that can time itself. util.info("[Client] Starting processes...") for iproc in xrange(n_procs): name = "Thread_{:02d}".format(iproc) thread = Thread(name=name, target=self._worker_thread, args=worker_args, kwargs=worker_kwargs) threads.append(thread) thread.start() thread.join(timeout=0.2) # Wait until all threads have completed. for thread in threads: thread.join() # Notifies the server that this client is done. activeq.task_done() util.info("[Client] Jobs complete.") sys.exit(0)
def manage_tasks(self, tasks): """Receives completed tasks from the clients and updates the tasks file. Args: tasks (dict): Tasks. """ t0 = Time.time() tasks_pth = path(self.params().get("tasks_pth", "")) n_tasks = len(tasks) n_done0 = n_tasks - self.taskq().qsize() n_done = n_done0 util.info("[Server] Tasks queued. {}/{} ({:.1f}%%) complete.".format( n_done, n_tasks, n_done / n_tasks * 100)) save_time = Time.time() # Set up a thread that joins self.taskq and only returns once # all the tasks have completed. The while loops continues as # long as taskq_thread is alive. taskq_thread = Thread(name="taskq", target=self._taskq_worker, args=(self.taskq,)) taskq_thread.start() while taskq_thread.is_alive(): # Wait for a done task to arrive. task = self.doneq().get() # Update the master tasks dict. task_name = task["task_name"] tasks[task_name].update(task) if tasks_pth and save_time.delta() > 10.: # Save task to disk. update_tasks_file(tasks_pth, tasks, overwrite=True) save_time = Time.time() # Report progress. n_done += 1 percent = float(n_done) / n_tasks * 100. dt = t0.delta() time_per_task = dt / float(n_done - n_done0) n_left = n_tasks - n_done t_left = Time(time_per_task * n_left) util.info("[Server] Task `{}` complete:\n\t\t {}/{} ({:.2f}%) {} " "\n\t\t Time left: {}.".format( task_name, n_done, n_tasks, percent, dt, t_left)) # Save tasks to disk one last time. update_tasks_file(tasks_pth, tasks, overwrite=True) # Wait for the clients to d/c. clients = [] while not self.activeq().empty(): clients.append(self.activeq().get()) util.debug("[Server] Waiting for clients to disconnect:\n\t{}.".format( "\n\t".join(clients))) self.activeq().join() time.sleep(0.5) self.activeq().close() self.taskq().close() self.doneq().close() util.info("[Server] Tasks completed.")
def serve_client(self, conn): util.debug('starting server thread to service %r', threading.current_thread().name) recv = conn.recv send = conn.send id_to_obj = self.id_to_obj while not self.stop: try: methodname = obj = None request = recv() ident, methodname, args, kwds = request obj, exposed, gettypeid = id_to_obj[ident] if methodname not in exposed: raise AttributeError('method %r of %r object is not in exposed=%r' % (methodname, type(obj), exposed)) function = getattr(obj, methodname) try: res = function(*args, **kwds) except Exception as e: msg = ('#ERROR', e) else: typeid = gettypeid and gettypeid.get(methodname, None) if typeid: rident, rexposed = self.create(conn, typeid, res) token = Token(typeid, self.address, rident) msg = ('#PROXY', (rexposed, token)) else: msg = ('#RETURN', res) except AttributeError: if methodname is None: msg = ('#TRACEBACK', format_exc()) else: try: fallback_func = self.fallback_mapping[methodname] result = fallback_func(self, conn, ident, obj, *args, **kwds) msg = ('#RETURN', result) except Exception: msg = ('#TRACEBACK', format_exc()) except EOFError: util.debug('got EOF -- exiting thread serving %r', threading.current_thread().name) sys.exit(0) except Exception: msg = ('#TRACEBACK', format_exc()) try: try: send(msg) except Exception as e: send(('#UNSERIALIZABLE', repr(msg))) except Exception as e: util.info('exception in thread serving %r', threading.current_thread().name) util.info(' ... message was %r', msg) util.info(' ... exception was %r', e) conn.close() sys.exit(1)
def _worker_thread(self, params, taskq, doneq, args, kwargs, timeout=1e5): """Target function to be run in a thread, which manages the job process. Args: params (dict): Parameters. taskq (multiprocessing.JoinableQueue): Shared queue for tasks. doneq (multiprocessing.Queue): Shared queue for done tasks. args (tuple): Arguments for ``_worker_process``. kwargs (dict): Keyword arguments for ``_worker_process``. Keyword Args: timeout (float): Time to run process before signaling that it be restarted. This circumvents memory leaks. """ util.info("[Client] Starting thread: {}.".format(current_thread())) finish = Event() # Loop until taskq is empty. while True: try: if taskq.empty(): break except (EOFError, IOError): break finish.clear() # Spawn a process. proc = self._spawn(finish, params, taskq, doneq, args, kwargs) proc.start() # Wait for timeout (or until the proc finishes), then # signal it to finish and wait until it returns. proc.join(timeout=timeout) finish.set() proc.join() # Check that the exit was not an error. if proc.exitcode == 100: util.info("[Client] Process killed, restarting.") elif proc.exitcode > 0: raise ProcError(proc) util.info("[Client] Ending thread: {}.".format(current_thread())) sys.exit(0)
if not e.args: exitcode = 1 elif type(e.args[0]) is int: exitcode = e.args[0] else: sys.stderr.write(e.args[0] + '\n') sys.stderr.flush() exitcode = 1 except: exitcode = 1 import traceback sys.stderr.write('Process %s:\n' % self.name) sys.stderr.flush() traceback.print_exc() util.info('process exiting with exitcode %d' % exitcode) return exitcode # # We subclass bytes to avoid accidental transmission of auth keys over network # class AuthenticationString(bytes): def __reduce__(self): from multiprocessing.forking import Popen if not Popen.thread_is_spawning(): raise TypeError('Pickling an AuthenticationString object is ' 'disallowed for security reasons') return AuthenticationString, (bytes(self), )
except EOFError: util.debug('got EOF -- exiting thread serving %r', threading.current_thread().name) sys.exit(0) except Exception: msg = ('#TRACEBACK', format_exc()) try: try: send(msg) except Exception, e: send(('#UNSERIALIZABLE', repr(msg))) except Exception, e: util.info('exception in thread serving %r', threading.current_thread().name) util.info(' ... message was %r', msg) util.info(' ... exception was %r', e) conn.close() sys.exit(1) def fallback_getvalue(self, conn, ident, obj): return obj def fallback_str(self, conn, ident, obj): return str(obj) def fallback_repr(self, conn, ident, obj): return repr(obj) fallback_mapping = {
def serve_client(self, conn): util.debug('starting server thread to service %r', threading.current_thread().name) recv = conn.recv send = conn.send id_to_obj = self.id_to_obj while not self.stop: try: methodname = obj = None request = recv() ident, methodname, args, kwds = request obj, exposed, gettypeid = id_to_obj[ident] if methodname not in exposed: raise AttributeError( 'method %r of %r object is not in exposed=%r' % (methodname, type(obj), exposed)) function = getattr(obj, methodname) try: res = function(*args, **kwds) except Exception as e: msg = ('#ERROR', e) else: typeid = gettypeid and gettypeid.get(methodname, None) if typeid: rident, rexposed = self.create(conn, typeid, res) token = Token(typeid, self.address, rident) msg = ('#PROXY', (rexposed, token)) else: msg = ('#RETURN', res) except AttributeError: if methodname is None: msg = ('#TRACEBACK', format_exc()) else: try: fallback_func = self.fallback_mapping[methodname] result = fallback_func(self, conn, ident, obj, *args, **kwds) msg = ('#RETURN', result) except Exception: msg = ('#TRACEBACK', format_exc()) except EOFError: util.debug('got EOF -- exiting thread serving %r', threading.current_thread().name) sys.exit(0) except Exception: msg = ('#TRACEBACK', format_exc()) try: try: send(msg) except Exception as e: send(('#UNSERIALIZABLE', repr(msg))) except Exception as e: util.info('exception in thread serving %r', threading.current_thread().name) util.info(' ... message was %r', msg) util.info(' ... exception was %r', e) conn.close() sys.exit(1) return