示例#1
0
    def __init__(self,
                 conf,
                 name=None,
                 child_type="worker",
                 age=0,
                 ppid=0,
                 timeout=30):

        if name is None:
            name = self.__class__.__name__
        self.name = name

        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout
        self.conf = conf

        # initialize
        self.booted = False
        self.alive = True
        self.debug = self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)

        self.on_init(self.conf)
示例#2
0
    def __init__(self, args, specs=[], name=None,
            child_type="supervisor", age=0, ppid=0,
            timeout=30):

        # set conf
        conf = DEFAULT_CONF.copy()
        conf.update(args)
        self.conf = conf


        specs.extend(self.on_init(conf))

        for spec in specs:
            c = Child(*spec)
            self._CHILDREN_SPECS.append(c)
            self._SPECS_BYNAME[c.name] = c


        if name is None:
            name =  self.__class__.__name__
        self.name = name
        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout


        self.pid = None
        self.num_children = len(self._CHILDREN_SPECS)
        self.child_age = 0
        self.booted = False
        self.stopping = False
        self.debug =self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)
示例#3
0
    def __init__(self,
                 args,
                 specs=[],
                 name=None,
                 child_type="supervisor",
                 age=0,
                 ppid=0,
                 timeout=30):

        # set conf
        conf = DEFAULT_CONF.copy()
        conf.update(args)
        self.conf = conf

        specs.extend(self.on_init(conf))

        for spec in specs:
            c = Child(*spec)
            self._CHILDREN_SPECS.append(c)
            self._SPECS_BYNAME[c.name] = c

        if name is None:
            name = self.__class__.__name__
        self.name = name
        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout

        self.pid = None
        self.num_children = len(self._CHILDREN_SPECS)
        self.child_age = 0
        self.booted = False
        self.stopping = False
        self.debug = self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)
示例#4
0
    def __init__(self, conf, name=None, child_type="worker", age=0, ppid=0, timeout=30):

        if name is None:
            name = self.__class__.__name__
        self.name = name

        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout
        self.conf = conf

        # initialize
        self.booted = False
        self.alive = True
        self.debug = self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)

        self.on_init(self.conf)
示例#5
0
文件: pool.py 项目: menghan/pistil
    def __init__(self,
                 args,
                 spec=(),
                 name=None,
                 child_type="supervisor",
                 age=0,
                 ppid=0,
                 timeout=30):

        if not isinstance(spec, tuple):
            raise TypeError("spec should be a tuple")

        # set conf
        conf = DEFAULT_CONF.copy()
        conf.update(args)
        self.conf = conf

        # set number of workers
        self.num_workers = conf.get('num_workers', 1)

        ret = self.on_init(conf)
        if not ret:
            self._SPEC = Child(*spec)
        else:
            self._SPEC = Child(*ret)

        if name is None:
            name = self.__class__.__name__

        self.name = name
        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout

        self.pid = None
        self.child_age = 0
        self.booted = False
        self.stopping = False
        self.debug = self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)
示例#6
0
class Worker(object):

    _SIGNALS = map(lambda x: getattr(signal, "SIG%s" % x), "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split())

    _PIPE = []

    def __init__(self, conf, name=None, child_type="worker", age=0, ppid=0, timeout=30):

        if name is None:
            name = self.__class__.__name__
        self.name = name

        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout
        self.conf = conf

        # initialize
        self.booted = False
        self.alive = True
        self.debug = self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)

        self.on_init(self.conf)

    def on_init(self, conf):
        pass

    def pid(self):
        return os.getpid()

    pid = util.cached_property(pid)

    def notify(self):
        """\
        Your worker subclass must arrange to have this method called
        once every ``self.timeout`` seconds. If you fail in accomplishing
        this task, the master process will murder your workers.
        """
        self.tmp.notify()

    def handle(self):
        raise NotImplementedError

    def run(self):
        """\
        This is the mainloop of a worker process. You should override
        this method in a subclass to provide the intended behaviour
        for your particular evil schemes.
        """
        while True:
            self.notify()
            self.handle()
            time.sleep(0.1)

    def on_init_process(self):
        """ method executed when we init a process """
        pass

    def init_process(self):
        """\
        If you override this method in a subclass, the last statement
        in the function should be to call this method with
        super(MyWorkerClass, self).init_process() so that the ``run()``
        loop is initiated.
        """
        util.set_owner_process(self.conf.get("uid", os.geteuid()), self.conf.get("gid", os.getegid()))

        # Reseed the random number generator
        util.seed()

        # For waking ourselves up
        self._PIPE = os.pipe()
        map(util.set_non_blocking, self._PIPE)
        map(util.close_on_exec, self._PIPE)

        # Prevent fd inherientence
        util.close_on_exec(self.tmp.fileno())
        self.init_signals()

        self.on_init_process()

        # Enter main run loop
        self.booted = True
        self.run()

    def init_signals(self):
        map(lambda s: signal.signal(s, signal.SIG_DFL), self._SIGNALS)
        signal.signal(signal.SIGQUIT, self.handle_quit)
        signal.signal(signal.SIGTERM, self.handle_exit)
        signal.signal(signal.SIGINT, self.handle_exit)
        signal.signal(signal.SIGWINCH, self.handle_winch)

    def handle_quit(self, sig, frame):
        self.alive = False

    def handle_exit(self, sig, frame):
        self.alive = False
        sys.exit(0)

    def handle_winch(self, sig, fname):
        # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD.
        return
示例#7
0
class Arbiter(object):
    """
    Arbiter maintain the workers processes alive. It launches or
    kills them if needed. It also manages application reloading
    via SIGHUP/USR2.
    """

    _SPECS_BYNAME = {}
    _CHILDREN_SPECS = []

    # A flag indicating if a worker failed to
    # to boot. If a worker process exist with
    # this error code, the arbiter will terminate.
    _WORKER_BOOT_ERROR = 3

    _WORKERS = {}    
    _PIPE = []

    # I love dynamic languages
    _SIG_QUEUE = []
    _SIGNALS = map(
        lambda x: getattr(signal, "SIG%s" % x),
        "HUP QUIT INT TERM USR1 WINCH".split()
    )
    _SIG_NAMES = dict(
        (getattr(signal, name), name[3:].lower()) for name in dir(signal)
        if name[:3] == "SIG" and name[3] != "_"
    )
    
    def __init__(self, args, specs=[], name=None,
            child_type="supervisor", age=0, ppid=0,
            timeout=30):

        # set conf
        conf = DEFAULT_CONF.copy()
        conf.update(args)
        self.conf = conf


        specs.extend(self.on_init(conf))

        for spec in specs:
            c = Child(*spec)
            self._CHILDREN_SPECS.append(c)
            self._SPECS_BYNAME[c.name] = c


        if name is None:
            name =  self.__class__.__name__
        self.name = name
        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout


        self.pid = None
        self.num_children = len(self._CHILDREN_SPECS)
        self.child_age = 0
        self.booted = False
        self.stopping = False
        self.debug =self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)
        
    def on_init(self, args):
        return []


    def on_init_process(self):
        """ method executed when we init a process """
        pass


    def init_process(self):
        """\
        If you override this method in a subclass, the last statement
        in the function should be to call this method with
        super(MyWorkerClass, self).init_process() so that the ``run()``
        loop is initiated.
        """

        # set current pid
        self.pid = os.getpid()
        
        util.set_owner_process(self.conf.get("uid", os.geteuid()),
                self.conf.get("gid", os.getegid()))

        # Reseed the random number generator
        util.seed()

        # prevent fd inheritance
        util.close_on_exec(self.tmp.fileno())

         # init signals
        self.init_signals()

        util._setproctitle("arbiter [%s]" % self.name)
        self.on_init_process()

        log.debug("Arbiter %s booted on %s", self.name, self.pid)
        self.when_ready()
        # Enter main run loop
        self.booted = True
        self.run()
    

    def when_ready(self):
        pass

    def init_signals(self):
        """\
        Initialize master signal handling. Most of the signals
        are queued. Child signals only wake up the master.
        """
        if self._PIPE:
            map(os.close, self._PIPE)
        self._PIPE = pair = os.pipe()
        map(util.set_non_blocking, pair)
        map(util.close_on_exec, pair)
        map(lambda s: signal.signal(s, self.signal), self._SIGNALS)
        signal.signal(signal.SIGCHLD, self.handle_chld)

    def signal(self, sig, frame):
        if len(self._SIG_QUEUE) < 5:
            self._SIG_QUEUE.append(sig)
            self.wakeup()
        else:
            log.warn("Dropping signal: %s", sig)

    def run(self):
        "Main master loop." 
        if not self.booted:
            return self.init_process()

        self.spawn_workers()
        while True:
            try:
                # notfy the master
                self.tmp.notify()
                self.reap_workers()
                sig = self._SIG_QUEUE.pop(0) if len(self._SIG_QUEUE) else None
                if sig is None:
                    self.sleep()
                    self.murder_workers()
                    self.manage_workers()
                    continue
                
                if sig not in self._SIG_NAMES:
                    log.info("Ignoring unknown signal: %s", sig)
                    continue
                
                signame = self._SIG_NAMES.get(sig)
                handler = getattr(self, "handle_%s" % signame, None)
                if not handler:
                    log.error("Unhandled signal: %s", signame)
                    continue
                log.info("Handling signal: %s", signame)
                handler()
                self.tmp.notify()
                self.wakeup()
            except StopIteration:
                self.halt()
            except KeyboardInterrupt:
                self.halt()
            except HaltServer, inst:
                self.halt(reason=inst.reason, exit_status=inst.exit_status)
            except SystemExit:
                raise
            except Exception:
                log.info("Unhandled exception in main loop:\n%s",  
                            traceback.format_exc())
                self.stop(False)
                sys.exit(-1)
示例#8
0
class Arbiter(object):
    """
    Arbiter maintain the workers processes alive. It launches or
    kills them if needed. It also manages application reloading
    via SIGHUP/USR2.
    """

    _SPECS_BYNAME = {}
    _CHILDREN_SPECS = []

    # A flag indicating if a worker failed to
    # to boot. If a worker process exist with
    # this error code, the arbiter will terminate.
    _WORKER_BOOT_ERROR = 3

    _WORKERS = {}
    _PIPE = []

    # I love dynamic languages
    _SIG_QUEUE = []
    _SIGNALS = map(lambda x: getattr(signal, "SIG%s" % x),
                   "HUP QUIT INT TERM USR1 WINCH".split())
    _SIG_NAMES = dict((getattr(signal, name), name[3:].lower())
                      for name in dir(signal)
                      if name[:3] == "SIG" and name[3] != "_")

    def __init__(self,
                 args,
                 specs=[],
                 name=None,
                 child_type="supervisor",
                 age=0,
                 ppid=0,
                 timeout=30):

        # set conf
        conf = DEFAULT_CONF.copy()
        conf.update(args)
        self.conf = conf

        specs.extend(self.on_init(conf))

        for spec in specs:
            c = Child(*spec)
            self._CHILDREN_SPECS.append(c)
            self._SPECS_BYNAME[c.name] = c

        if name is None:
            name = self.__class__.__name__
        self.name = name
        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout

        self.pid = None
        self.num_children = len(self._CHILDREN_SPECS)
        self.child_age = 0
        self.booted = False
        self.stopping = False
        self.debug = self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)

    def on_init(self, args):
        return []

    def on_init_process(self):
        """ method executed when we init a process """
        pass

    def init_process(self):
        """\
        If you override this method in a subclass, the last statement
        in the function should be to call this method with
        super(MyWorkerClass, self).init_process() so that the ``run()``
        loop is initiated.
        """

        # set current pid
        self.pid = os.getpid()

        util.set_owner_process(self.conf.get("uid", os.geteuid()),
                               self.conf.get("gid", os.getegid()))

        # Reseed the random number generator
        util.seed()

        # prevent fd inheritance
        util.close_on_exec(self.tmp.fileno())

        # init signals
        self.init_signals()

        util._setproctitle("arbiter [%s]" % self.name)
        self.on_init_process()

        log.debug("Arbiter %s booted on %s", self.name, self.pid)
        self.when_ready()
        # Enter main run loop
        self.booted = True
        self.run()

    def when_ready(self):
        pass

    def init_signals(self):
        """\
        Initialize master signal handling. Most of the signals
        are queued. Child signals only wake up the master.
        """
        if self._PIPE:
            map(os.close, self._PIPE)
        self._PIPE = pair = os.pipe()
        map(util.set_non_blocking, pair)
        map(util.close_on_exec, pair)
        map(lambda s: signal.signal(s, self.signal), self._SIGNALS)
        signal.signal(signal.SIGCHLD, self.handle_chld)

    def signal(self, sig, frame):
        if len(self._SIG_QUEUE) < 5:
            self._SIG_QUEUE.append(sig)
            self.wakeup()
        else:
            log.warn("Dropping signal: %s", sig)

    def run(self):
        "Main master loop."
        if not self.booted:
            return self.init_process()

        self.spawn_workers()
        while True:
            try:
                # notfy the master
                self.tmp.notify()
                self.reap_workers()
                sig = self._SIG_QUEUE.pop(0) if len(self._SIG_QUEUE) else None
                if sig is None:
                    self.sleep()
                    self.murder_workers()
                    self.manage_workers()
                    continue

                if sig not in self._SIG_NAMES:
                    log.info("Ignoring unknown signal: %s", sig)
                    continue

                signame = self._SIG_NAMES.get(sig)
                handler = getattr(self, "handle_%s" % signame, None)
                if not handler:
                    log.error("Unhandled signal: %s", signame)
                    continue
                log.info("Handling signal: %s", signame)
                handler()
                self.tmp.notify()
                self.wakeup()
            except StopIteration:
                self.halt()
            except KeyboardInterrupt:
                self.halt()
            except HaltServer, inst:
                self.halt(reason=inst.reason, exit_status=inst.exit_status)
            except SystemExit:
                raise
            except Exception:
                log.info("Unhandled exception in main loop:\n%s",
                         traceback.format_exc())
                self.stop(False)
                sys.exit(-1)
示例#9
0
文件: arbiter.py 项目: derkan/pistil
class Arbiter(object):
    """
    Arbiter maintain the workers processes alive. It launches or
    kills them if needed. It also manages application reloading
    via SIGHUP/USR2.
    """

    _SPECS_BYNAME = {}
    _CHILDREN_SPECS = []

    # A flag indicating if a worker failed to
    # to boot. If a worker process exist with
    # this error code, the arbiter will terminate.
    _WORKER_BOOT_ERROR = 3

    _WORKERS = {}    
    _PIPE = []

    # I love dynamic languages
    _SIG_QUEUE = []
    _SIGNALS = map(
        lambda x: getattr(signal, "SIG%s" % x),
        "HUP QUIT INT TERM USR1 WINCH".split()
    )
    _SIG_NAMES = dict(
        (getattr(signal, name), name[3:].lower()) for name in dir(signal)
        if name[:3] == "SIG" and name[3] != "_"
    )
    
    def __init__(self, args, specs=[], name=None,
            child_type="supervisor", age=0, ppid=0,
            timeout=30):

        # set conf
        conf = DEFAULT_CONF.copy()
        conf.update(args)
        self.conf = conf


        specs.extend(self.on_init(conf))

        for spec in specs:
            c = Child(*spec)
            self._CHILDREN_SPECS.append(c)
            self._SPECS_BYNAME[c.name] = c


        if name is None:
            name =  self.__class__.__name__
        self.name = name
        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout


        self.pid = None
        self.num_children = len(self._CHILDREN_SPECS)
        self.child_age = 0
        self.booted = False
        self.stopping = False
        self.debug =self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)
        
    def on_init(self, args):
        return []


    def on_init_process(self):
        """ method executed when we init a process """
        pass


    def init_process(self):
        """\
        If you override this method in a subclass, the last statement
        in the function should be to call this method with
        super(MyWorkerClass, self).init_process() so that the ``run()``
        loop is initiated.
        """

        # set current pid
        self.pid = os.getpid()
        
        util.set_owner_process(self.conf.get("uid", os.geteuid()),
                self.conf.get("gid", os.getegid()))

        # Reseed the random number generator
        util.seed()

        # prevent fd inheritance
        util.close_on_exec(self.tmp.fileno())

         # init signals
        self.init_signals()

        util._setproctitle("arbiter [%s]" % self.name)
        self.on_init_process()

        log.debug("Arbiter %s booted on %s", self.name, self.pid)
        self.when_ready()
        # Enter main run loop
        self.booted = True
        self.run()
    

    def when_ready(self):
        pass

    def init_signals(self):
        """\
        Initialize master signal handling. Most of the signals
        are queued. Child signals only wake up the master.
        """
        if self._PIPE:
            map(os.close, self._PIPE)
        self._PIPE = pair = os.pipe()
        map(util.set_non_blocking, pair)
        map(util.close_on_exec, pair)
        map(lambda s: signal.signal(s, self.signal), self._SIGNALS)
        signal.signal(signal.SIGCHLD, self.handle_chld)

    def signal(self, sig, frame):
        if len(self._SIG_QUEUE) < 5:
            self._SIG_QUEUE.append(sig)
            self.wakeup()
        else:
            log.warn("Dropping signal: %s", sig)

    def run(self):
        "Main master loop." 
        if not self.booted:
            return self.init_process()

        self.spawn_workers()
        while True:
            try:
                # notfy the master
                self.tmp.notify()
                self.reap_workers()
                sig = self._SIG_QUEUE.pop(0) if len(self._SIG_QUEUE) else None
                if sig is None:
                    self.sleep()
                    self.murder_workers()
                    self.manage_workers()
                    continue
                
                if sig not in self._SIG_NAMES:
                    log.info("Ignoring unknown signal: %s", sig)
                    continue
                
                signame = self._SIG_NAMES.get(sig)
                handler = getattr(self, "handle_%s" % signame, None)
                if not handler:
                    log.error("Unhandled signal: %s", signame)
                    continue
                log.info("Handling signal: %s", signame)
                handler()
                self.tmp.notify()
                self.wakeup()
            except StopIteration:
                self.halt()
            except KeyboardInterrupt:
                self.halt()
            except HaltServer as inst:
                self.halt(reason=inst.reason, exit_status=inst.exit_status)
            except SystemExit:
                raise
            except Exception:
                log.info("Unhandled exception in main loop:\n%s",  
                            traceback.format_exc())
                self.stop(False)
                sys.exit(-1)

    def handle_chld(self, sig, frame):
        "SIGCHLD handling"
        self.wakeup()
        self.reap_workers()
        
    def handle_hup(self):
        """\
        HUP handling.
        - Reload configuration
        - Start the new worker processes with a new configuration
        - Gracefully shutdown the old worker processes
        """
        log.info("Hang up: %s", self.name)
        self.reload()
        
    def handle_quit(self):
        "SIGQUIT handling"
        raise StopIteration
    
    def handle_int(self):
        "SIGINT handling"
        raise StopIteration
    
    def handle_term(self):
        "SIGTERM handling"
        self.stop(False)
        raise StopIteration

    def handle_usr1(self):
        """\
        SIGUSR1 handling.
        Kill all workers by sending them a SIGUSR1
        """
        self.kill_workers(signal.SIGUSR1)
    
    def handle_winch(self):
        "SIGWINCH handling"
        if os.getppid() == 1 or os.getpgrp() != os.getpid():
            log.info("graceful stop of workers")
            self.num_workers = 0
            self.kill_workers(signal.SIGQUIT)
        else:
            log.info("SIGWINCH ignored. Not daemonized")
    
    def wakeup(self):
        """\
        Wake up the arbiter by writing to the _PIPE
        """
        try:
            os.write(self._PIPE[1], '.')
        except IOError as e:
            if e.errno not in [errno.EAGAIN, errno.EINTR]:
                raise
        
                    
    def halt(self, reason=None, exit_status=0):
        """ halt arbiter """
        log.info("Shutting down: %s", self.name)
        if reason is not None:
            log.info("Reason: %s", reason)
        self.stop()
        log.info("See you next")
        sys.exit(exit_status)
        
    def sleep(self):
        """\
        Sleep until _PIPE is readable or we timeout.
        A readable _PIPE means a signal occurred.
        """
        try:
            ready = select.select([self._PIPE[0]], [], [], 1.0)
            if not ready[0]:
                return
            while os.read(self._PIPE[0], 1):
                pass
        except select.error as e:
            if hasattr(e, 'errno'):
                v_err = e.errno
            else:
                v_err = e[0]
            if v_err not in [errno.EAGAIN, errno.EINTR]:
                raise
        except OSError as e:
            if hasattr(e, 'errno'):
                v_err = e.errno
            else:
                v_err = e[0]
            if v_err not in [errno.EAGAIN, errno.EINTR]:
                raise
        except KeyboardInterrupt:
            sys.exit()
            
    
    def on_stop(self, graceful=True):
        """ method used to pass code when the server start """

    def stop(self, graceful=True):
        """\
        Stop workers
        
        :attr graceful: boolean, If True (the default) workers will be
        killed gracefully  (ie. trying to wait for the current connection)
        """
        
        ## pass any actions before we effectively stop
        self.on_stop(graceful=graceful)
        self.stopping = True
        sig = signal.SIGQUIT
        if not graceful:
            sig = signal.SIGTERM
        limit = time.time() + self.timeout
        while True:
            if time.time() >= limit or not self._WORKERS:
                break
            self.kill_workers(sig)
            time.sleep(0.1)
            self.reap_workers()
        self.kill_workers(signal.SIGKILL)   
        self.stopping = False

    def on_reload(self):
        """ method executed on reload """


    def reload(self):
        """ 
        used on HUP
        """
    
        # exec on reload hook
        self.on_reload()

        OLD__WORKERS = self._WORKERS.copy()

        # don't kill
        to_reload = []

        # spawn new workers with new app & conf
        for child in self._CHILDREN_SPECS:
            if child.child_type != "supervisor":
                to_reload.append(child)

        # set new proc_name
        util._setproctitle("arbiter [%s]" % self.name)
        
        # kill old workers
        for wpid, (child, state) in OLD__WORKERS.items():
            if state and child.timeout is not None:
                if child.child_type == "supervisor":
                    # we only reload suprvisors.
                    sig = signal.SIGHUP
                elif child.child_type == "brutal_kill":
                    sig =  signal.SIGTERM
                else:
                    sig =  signal.SIGQUIT
                self.kill_worker(wpid, sig)

        
    def murder_workers(self):
        """\
        Kill unused/idle workers
        """
        for (pid, child_info) in self._WORKERS.items():
            (child, state) = child_info
            if state and child.timeout is not None:
                try:
                    diff = time.time() - os.fstat(child.tmp.fileno()).st_ctime
                    if diff <= child.timeout:
                        continue
                except ValueError:
                    continue
            elif state and child.timeout is None:
                continue

            log.critical("WORKER TIMEOUT (pid:%s)", pid)
            self.kill_worker(pid, signal.SIGKILL)
        
    def reap_workers(self):
        """\
        Reap workers to avoid zombie processes
        """
        try:
            while True:
                wpid, status = os.waitpid(-1, os.WNOHANG)
                if not wpid:
                    break
                
                # A worker said it cannot boot. We'll shutdown
                # to avoid infinite start/stop cycles.
                exitcode = status >> 8
                if exitcode == self._WORKER_BOOT_ERROR:
                    reason = "Worker failed to boot."
                    raise HaltServer(reason, self._WORKER_BOOT_ERROR)
                child_info = self._WORKERS.pop(wpid, None)

                if not child_info:
                    continue

                child, state = child_info
                child.tmp.close()
                if child.child_type in RESTART_WORKERS and not self.stopping:
                    self._WORKERS["<killed %s>"  % id(child)] = (child, 0)
        except OSError as e:
            if hasattr(e, 'errno'):
                v_err = e.errno
            else:
                v_err = e[0]
            if v_err == errno.ECHILD:
                pass
    
    def manage_workers(self):
        """\
        Maintain the number of workers by spawning or killing
        as required.
        """

        for pid, (child, state) in self._WORKERS.items():
            if not state:
                del self._WORKERS[pid]
                self.spawn_child(self._SPECS_BYNAME[child.name])

    def pre_fork(self, worker):
        """ methode executed on prefork """

    def post_fork(self, worker):
        """ method executed after we forked a worker """
            
    def spawn_child(self, child_spec):
        self.child_age += 1
        name = child_spec.name
        child_type = child_spec.child_type

        child_args = self.conf
        child_args.update(child_spec.args)

        try:
            # initialize child class
            child = child_spec.child_class(
                        child_args,
                        name = name,
                        child_type = child_type, 
                        age = self.child_age,
                        ppid = self.pid,
                        timeout = child_spec.timeout)
        except:
            log.info("Unhandled exception while creating '%s':\n%s",  
                            name, traceback.format_exc())
            return


        self.pre_fork(child)
        pid = os.fork()
        if pid != 0:
            self._WORKERS[pid] = (child, 1)
            return

        # Process Child
        worker_pid = os.getpid()
        try:
            util._setproctitle("worker %s [%s]" % (name,  worker_pid))
            log.info("Booting %s (%s) with pid: %s", name,
                    child_type, worker_pid)
            self.post_fork(child)
            child.init_process()
            sys.exit(0)
        except SystemExit:
            raise
        except:
            log.exception("Exception in worker process:")
            if not child.booted:
                sys.exit(self._WORKER_BOOT_ERROR)
            sys.exit(-1)
        finally:
            log.info("Worker exiting (pid: %s)", worker_pid)
            try:
                child.tmp.close()
            except:
                pass

    def spawn_workers(self):
        """\
        Spawn new workers as needed.
        
        This is where a worker process leaves the main loop
        of the master process.
        """
        
        for child in self._CHILDREN_SPECS: 
            self.spawn_child(child)

    def kill_workers(self, sig):
        """\
        Lill all workers with the signal `sig`
        :attr sig: `signal.SIG*` value
        """
        for pid in self._WORKERS.keys():
            self.kill_worker(pid, sig)
                   

    def kill_worker(self, pid, sig):
        """\
        Kill a worker
        
        :attr pid: int, worker pid
        :attr sig: `signal.SIG*` value
         """
        if not isinstance(pid, int):
            return

        try:
            os.kill(pid, sig)
        except OSError as e:
            if hasattr(e, 'errno'):
                v_err = e.errno
            else:
                v_err = e[0]            
            if v_err == errno.ESRCH:
                try:
                    (child, info) = self._WORKERS.pop(pid)
                    child.tmp.close()
           
                    if not self.stopping:
                        self._WORKERS["<killed %s>"  % id(child)] = (child, 0)
                    return
                except (KeyError, OSError):
                    return
            raise            
示例#10
0
class Worker(object):

    _SIGNALS = map(lambda x: getattr(signal, "SIG%s" % x),
                   "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split())

    _PIPE = []

    def __init__(self,
                 conf,
                 name=None,
                 child_type="worker",
                 age=0,
                 ppid=0,
                 timeout=30):

        if name is None:
            name = self.__class__.__name__
        self.name = name

        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout
        self.conf = conf

        # initialize
        self.booted = False
        self.alive = True
        self.debug = self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)

        self.on_init(self.conf)

    def on_init(self, conf):
        pass

    def pid(self):
        return os.getpid()

    pid = util.cached_property(pid)

    def notify(self):
        """\
        Your worker subclass must arrange to have this method called
        once every ``self.timeout`` seconds. If you fail in accomplishing
        this task, the master process will murder your workers.
        """
        self.tmp.notify()

    def handle(self):
        raise NotImplementedError

    def run(self):
        """\
        This is the mainloop of a worker process. You should override
        this method in a subclass to provide the intended behaviour
        for your particular evil schemes.
        """
        while True:
            self.notify()
            self.handle()
            time.sleep(0.1)

    def on_init_process(self):
        """ method executed when we init a process """
        pass

    def init_process(self):
        """\
        If you override this method in a subclass, the last statement
        in the function should be to call this method with
        super(MyWorkerClass, self).init_process() so that the ``run()``
        loop is initiated.
        """
        util.set_owner_process(self.conf.get("uid", os.geteuid()),
                               self.conf.get("gid", os.getegid()))

        # Reseed the random number generator
        util.seed()

        # For waking ourselves up
        self._PIPE = os.pipe()
        map(util.set_non_blocking, self._PIPE)
        map(util.close_on_exec, self._PIPE)

        # Prevent fd inherientence
        util.close_on_exec(self.tmp.fileno())
        self.init_signals()

        self.on_init_process()

        # Enter main run loop
        self.booted = True
        self.run()

    def init_signals(self):
        map(lambda s: signal.signal(s, signal.SIG_DFL), self._SIGNALS)
        signal.signal(signal.SIGQUIT, self.handle_quit)
        signal.signal(signal.SIGTERM, self.handle_exit)
        signal.signal(signal.SIGINT, self.handle_exit)
        signal.signal(signal.SIGWINCH, self.handle_winch)

    def handle_quit(self, sig, frame):
        self.alive = False

    def handle_exit(self, sig, frame):
        self.alive = False
        sys.exit(0)

    def handle_winch(self, sig, fname):
        # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD.
        return
示例#11
0
class Arbiter(object):
    """
    Arbiter maintain the workers processes alive. It launches or
    kills them if needed. It also manages application reloading
    via SIGHUP/USR2.
    """

    _SPECS_BYNAME = {}
    _CHILDREN_SPECS = []

    # A flag indicating if a worker failed to
    # to boot. If a worker process exist with
    # this error code, the arbiter will terminate.
    _WORKER_BOOT_ERROR = 3

    _WORKERS = {}
    _PIPE = []

    # I love dynamic languages
    _SIG_QUEUE = []
    _SIGNALS = map(lambda x: getattr(signal, "SIG%s" % x),
                   "HUP QUIT INT TERM USR1 WINCH".split())
    _SIG_NAMES = dict((getattr(signal, name), name[3:].lower())
                      for name in dir(signal)
                      if name[:3] == "SIG" and name[3] != "_")

    def __init__(self,
                 args,
                 specs=[],
                 name=None,
                 child_type="supervisor",
                 age=0,
                 ppid=0,
                 timeout=30):

        # set conf
        conf = DEFAULT_CONF.copy()
        conf.update(args)
        self.conf = conf

        specs.extend(self.on_init(conf))

        for spec in specs:
            c = Child(*spec)
            self._CHILDREN_SPECS.append(c)
            self._SPECS_BYNAME[c.name] = c

        if name is None:
            name = self.__class__.__name__
        self.name = name
        self.child_type = child_type
        self.age = age
        self.ppid = ppid
        self.timeout = timeout

        self.pid = None
        self.num_children = len(self._CHILDREN_SPECS)
        self.child_age = 0
        self.booted = False
        self.stopping = False
        self.debug = self.conf.get("debug", False)
        self.tmp = WorkerTmp(self.conf)

    def on_init(self, args):
        return []

    def on_init_process(self):
        """ method executed when we init a process """
        pass

    def init_process(self):
        """\
        If you override this method in a subclass, the last statement
        in the function should be to call this method with
        super(MyWorkerClass, self).init_process() so that the ``run()``
        loop is initiated.
        """

        # set current pid
        self.pid = os.getpid()

        util.set_owner_process(self.conf.get("uid", os.geteuid()),
                               self.conf.get("gid", os.getegid()))

        # Reseed the random number generator
        util.seed()

        # prevent fd inheritance
        util.close_on_exec(self.tmp.fileno())

        # init signals
        self.init_signals()

        util._setproctitle("arbiter [%s]" % self.name)
        self.on_init_process()

        log.debug("Arbiter %s booted on %s", self.name, self.pid)
        self.when_ready()
        # Enter main run loop
        self.booted = True
        self.run()

    def when_ready(self):
        pass

    def init_signals(self):
        """\
        Initialize master signal handling. Most of the signals
        are queued. Child signals only wake up the master.
        """
        if self._PIPE:
            map(os.close, self._PIPE)
        self._PIPE = pair = os.pipe()
        map(util.set_non_blocking, pair)
        map(util.close_on_exec, pair)
        map(lambda s: signal.signal(s, self.signal), self._SIGNALS)
        signal.signal(signal.SIGCHLD, self.handle_chld)

    def signal(self, sig, frame):
        if len(self._SIG_QUEUE) < 5:
            self._SIG_QUEUE.append(sig)
            self.wakeup()
        else:
            log.warn("Dropping signal: %s", sig)

    def run(self):
        "Main master loop."
        if not self.booted:
            return self.init_process()

        self.spawn_workers()
        while True:
            try:
                # notfy the master
                self.tmp.notify()
                self.reap_workers()
                sig = self._SIG_QUEUE.pop(0) if len(self._SIG_QUEUE) else None
                if sig is None:
                    self.sleep()
                    self.murder_workers()
                    self.manage_workers()
                    continue

                if sig not in self._SIG_NAMES:
                    log.info("Ignoring unknown signal: %s", sig)
                    continue

                signame = self._SIG_NAMES.get(sig)
                handler = getattr(self, "handle_%s" % signame, None)
                if not handler:
                    log.error("Unhandled signal: %s", signame)
                    continue
                log.info("Handling signal: %s", signame)
                handler()
                self.tmp.notify()
                self.wakeup()
            except StopIteration:
                self.halt()
            except KeyboardInterrupt:
                self.halt()
            except HaltServer as inst:
                self.halt(reason=inst.reason, exit_status=inst.exit_status)
            except SystemExit:
                raise
            except Exception:
                log.info("Unhandled exception in main loop:\n%s",
                         traceback.format_exc())
                self.stop(False)
                sys.exit(-1)

    def handle_chld(self, sig, frame):
        "SIGCHLD handling"
        self.wakeup()
        self.reap_workers()

    def handle_hup(self):
        """\
        HUP handling.
        - Reload configuration
        - Start the new worker processes with a new configuration
        - Gracefully shutdown the old worker processes
        """
        log.info("Hang up: %s", self.name)
        self.reload()

    def handle_quit(self):
        "SIGQUIT handling"
        raise StopIteration

    def handle_int(self):
        "SIGINT handling"
        raise StopIteration

    def handle_term(self):
        "SIGTERM handling"
        self.stop(False)
        raise StopIteration

    def handle_usr1(self):
        """\
        SIGUSR1 handling.
        Kill all workers by sending them a SIGUSR1
        """
        self.kill_workers(signal.SIGUSR1)

    def handle_winch(self):
        "SIGWINCH handling"
        if os.getppid() == 1 or os.getpgrp() != os.getpid():
            log.info("graceful stop of workers")
            self.num_workers = 0
            self.kill_workers(signal.SIGQUIT)
        else:
            log.info("SIGWINCH ignored. Not daemonized")

    def wakeup(self):
        """\
        Wake up the arbiter by writing to the _PIPE
        """
        try:
            os.write(self._PIPE[1], '.')
        except IOError as e:
            if e.errno not in [errno.EAGAIN, errno.EINTR]:
                raise

    def halt(self, reason=None, exit_status=0):
        """ halt arbiter """
        log.info("Shutting down: %s", self.name)
        if reason is not None:
            log.info("Reason: %s", reason)
        self.stop()
        log.info("See you next")
        sys.exit(exit_status)

    def sleep(self):
        """\
        Sleep until _PIPE is readable or we timeout.
        A readable _PIPE means a signal occurred.
        """
        try:
            ready = select.select([self._PIPE[0]], [], [], 1.0)
            if not ready[0]:
                return
            while os.read(self._PIPE[0], 1):
                pass
        except select.error as e:
            if hasattr(e, 'errno'):
                v_err = e.errno
            else:
                v_err = e[0]
            if v_err not in [errno.EAGAIN, errno.EINTR]:
                raise
        except OSError as e:
            if hasattr(e, 'errno'):
                v_err = e.errno
            else:
                v_err = e[0]
            if v_err not in [errno.EAGAIN, errno.EINTR]:
                raise
        except KeyboardInterrupt:
            sys.exit()

    def on_stop(self, graceful=True):
        """ method used to pass code when the server start """

    def stop(self, graceful=True):
        """\
        Stop workers
        
        :attr graceful: boolean, If True (the default) workers will be
        killed gracefully  (ie. trying to wait for the current connection)
        """

        ## pass any actions before we effectively stop
        self.on_stop(graceful=graceful)
        self.stopping = True
        sig = signal.SIGQUIT
        if not graceful:
            sig = signal.SIGTERM
        limit = time.time() + self.timeout
        while True:
            if time.time() >= limit or not self._WORKERS:
                break
            self.kill_workers(sig)
            time.sleep(0.1)
            self.reap_workers()
        self.kill_workers(signal.SIGKILL)
        self.stopping = False

    def on_reload(self):
        """ method executed on reload """

    def reload(self):
        """ 
        used on HUP
        """

        # exec on reload hook
        self.on_reload()

        OLD__WORKERS = self._WORKERS.copy()

        # don't kill
        to_reload = []

        # spawn new workers with new app & conf
        for child in self._CHILDREN_SPECS:
            if child.child_type != "supervisor":
                to_reload.append(child)

        # set new proc_name
        util._setproctitle("arbiter [%s]" % self.name)

        # kill old workers
        for wpid, (child, state) in OLD__WORKERS.items():
            if state and child.timeout is not None:
                if child.child_type == "supervisor":
                    # we only reload suprvisors.
                    sig = signal.SIGHUP
                elif child.child_type == "brutal_kill":
                    sig = signal.SIGTERM
                else:
                    sig = signal.SIGQUIT
                self.kill_worker(wpid, sig)

    def murder_workers(self):
        """\
        Kill unused/idle workers
        """
        for (pid, child_info) in self._WORKERS.items():
            (child, state) = child_info
            if state and child.timeout is not None:
                try:
                    diff = time.time() - os.fstat(child.tmp.fileno()).st_ctime
                    if diff <= child.timeout:
                        continue
                except ValueError:
                    continue
            elif state and child.timeout is None:
                continue

            log.critical("WORKER TIMEOUT (pid:%s)", pid)
            self.kill_worker(pid, signal.SIGKILL)

    def reap_workers(self):
        """\
        Reap workers to avoid zombie processes
        """
        try:
            while True:
                wpid, status = os.waitpid(-1, os.WNOHANG)
                if not wpid:
                    break

                # A worker said it cannot boot. We'll shutdown
                # to avoid infinite start/stop cycles.
                exitcode = status >> 8
                if exitcode == self._WORKER_BOOT_ERROR:
                    reason = "Worker failed to boot."
                    raise HaltServer(reason, self._WORKER_BOOT_ERROR)
                child_info = self._WORKERS.pop(wpid, None)

                if not child_info:
                    continue

                child, state = child_info
                child.tmp.close()
                if child.child_type in RESTART_WORKERS and not self.stopping:
                    self._WORKERS["<killed %s>" % id(child)] = (child, 0)
        except OSError as e:
            if hasattr(e, 'errno'):
                v_err = e.errno
            else:
                v_err = e[0]
            if v_err == errno.ECHILD:
                pass

    def manage_workers(self):
        """\
        Maintain the number of workers by spawning or killing
        as required.
        """

        for pid, (child, state) in self._WORKERS.items():
            if not state:
                del self._WORKERS[pid]
                self.spawn_child(self._SPECS_BYNAME[child.name])

    def pre_fork(self, worker):
        """ methode executed on prefork """

    def post_fork(self, worker):
        """ method executed after we forked a worker """

    def spawn_child(self, child_spec):
        self.child_age += 1
        name = child_spec.name
        child_type = child_spec.child_type

        child_args = self.conf
        child_args.update(child_spec.args)

        try:
            # initialize child class
            child = child_spec.child_class(child_args,
                                           name=name,
                                           child_type=child_type,
                                           age=self.child_age,
                                           ppid=self.pid,
                                           timeout=child_spec.timeout)
        except:
            log.info("Unhandled exception while creating '%s':\n%s", name,
                     traceback.format_exc())
            return

        self.pre_fork(child)
        pid = os.fork()
        if pid != 0:
            self._WORKERS[pid] = (child, 1)
            return

        # Process Child
        worker_pid = os.getpid()
        try:
            util._setproctitle("worker %s [%s]" % (name, worker_pid))
            log.info("Booting %s (%s) with pid: %s", name, child_type,
                     worker_pid)
            self.post_fork(child)
            child.init_process()
            sys.exit(0)
        except SystemExit:
            raise
        except:
            log.exception("Exception in worker process:")
            if not child.booted:
                sys.exit(self._WORKER_BOOT_ERROR)
            sys.exit(-1)
        finally:
            log.info("Worker exiting (pid: %s)", worker_pid)
            try:
                child.tmp.close()
            except:
                pass

    def spawn_workers(self):
        """\
        Spawn new workers as needed.
        
        This is where a worker process leaves the main loop
        of the master process.
        """

        for child in self._CHILDREN_SPECS:
            self.spawn_child(child)

    def kill_workers(self, sig):
        """\
        Lill all workers with the signal `sig`
        :attr sig: `signal.SIG*` value
        """
        for pid in self._WORKERS.keys():
            self.kill_worker(pid, sig)

    def kill_worker(self, pid, sig):
        """\
        Kill a worker
        
        :attr pid: int, worker pid
        :attr sig: `signal.SIG*` value
         """
        if not isinstance(pid, int):
            return

        try:
            os.kill(pid, sig)
        except OSError as e:
            if hasattr(e, 'errno'):
                v_err = e.errno
            else:
                v_err = e[0]
            if v_err == errno.ESRCH:
                try:
                    (child, info) = self._WORKERS.pop(pid)
                    child.tmp.close()

                    if not self.stopping:
                        self._WORKERS["<killed %s>" % id(child)] = (child, 0)
                    return
                except (KeyError, OSError):
                    return
            raise