示例#1
0
    def __call__(self,
                 watchers,
                 controller=None,
                 pubsub_endpoint=None,
                 statsd=False,
                 stats_endpoint=None,
                 statsd_close_outputs=False,
                 multicast_endpoint=None,
                 env=None,
                 name=None,
                 context=None,
                 background=False,
                 stream_backend="thread",
                 plugins=None,
                 debug=False,
                 proc_name="circusd"):
        """Creates a Arbiter and a single watcher in it.

        Options:

          - **watchers** -- a list of watchers. A watcher in that case is a
                            dict containing:
            - **name** -- the name of the watcher (default: None)
            - **cmd** -- the command line used to run the Watcher.
            - **args** -- the args for the command (list or string).
            - **executable** -- When executable is given, the first item in
              the args sequence obtained from **cmd** is still treated by most
              programs as the command name, which can then be different from
              the actual executable name. It becomes the display name for the
              executing program in utilities such as **ps**.

            - **numprocesses** -- the number of processes to spawn
              (default: 1).
            - **warmup_delay** -- the delay in seconds between two spawns
              (default: 0)
            - **shell** -- if True, the processes are run in the shell
              (default: False)
            - **working_dir** - the working dir for the processes
              (default: None)
            - **uid** -- the user id used to run the processes (default: None)
            - **gid** -- the group id used to run the processes (default: None)
            - **env** -- the environment passed to the processes
              (default: None)
            - **send_hup**: if True, a process reload will be done by sending
              the SIGHUP signal. (default: False)
            - **stdout_stream**: a mapping containing the options for
              configuring
              the stdout stream. Default to None. When provided, may contain:

                - **class**: the fully qualified name of the class to use for
                             streaming. Defaults to circus.stream.FileStream
                - any other key will be passed the class constructor.
            - **stderr_stream**: a mapping containing the options for
              configuring the stderr stream. Default to None. When provided,
              may contain:

                - **class**: the fully qualified name of the class to use for
                  streaming. Defaults to circus.stream.FileStream
                - any other key will be passed the class constructor.
            - **max_retry**: the number of times we attempt to start a process,
              before we abandon and stop the whole watcher. (default: 5)
            - **hooks**: callback functions for hooking into the watcher
              startup and shutdown process. **hooks** is a dict where each key
              is the hook name and each value is a 2-tuple with the name of the
              callable or the callabled itself and a boolean flag indicating if
              an exception occuring in the hook should not be ignored.
              Possible values for the hook name: *before_start*, *after_start*,
              *before_stop*, *after_stop*, *before_signal*, *after_signal*

        - **controller** -- the zmq entry point
          (default: 'tcp://127.0.0.1:5555')
        - **pubsub_endpoint** -- the zmq entry point for the pubsub
          (default: 'tcp://127.0.0.1:5556')
        - **stats_endpoint** -- the stats endpoint. If not provided,
          the *circusd-stats* process will not be launched. (default: None)
        - **statsd_close_outputs** -- if True sends the circusd-stats
          stdout/stderr to /dev/null (default: False)
        - **context** -- the zmq context (default: None)
        - **background** -- If True, the arbiter is launched in a thread in the
          background (default: False)
        - **stream_backend** -- the backend that will be used for the streaming
          process. Can be *thread* or *gevent*. When set to *gevent* you need
          to have *gevent* and *gevent_zmq* installed. (default: thread)
        - **plugins** -- a list of plugins. Each item is a mapping with:

            - **use** -- Fully qualified name that points to the plugin class
            - every other value is passed to the plugin in the **config**
              option

        - **debug** -- If True the arbiter is launched in debug mode
          (default: False)
        - **proc_name** -- the arbiter process name (default: circusd)
        """
        from circus.util import (DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
                                 DEFAULT_ENDPOINT_MULTICAST,
                                 DEFAULT_ENDPOINT_STATS)
        if controller is None:
            controller = DEFAULT_ENDPOINT_DEALER
        if pubsub_endpoint is None:
            pubsub_endpoint = DEFAULT_ENDPOINT_SUB
        if multicast_endpoint is None:
            multicast_endpoint = DEFAULT_ENDPOINT_MULTICAST
        if stats_endpoint is None and statsd:
            stats_endpoint = DEFAULT_ENDPOINT_STATS
        elif stats_endpoint is not None and not statsd:
            warnings.warn(
                "You defined a stats_endpoint without "
                "setting up statsd to True.", DeprecationWarning)
            statsd = True

        from circus.watcher import Watcher
        Arbiter = self._get_arbiter_klass(background=background)

        _watchers = []

        for watcher in watchers:
            cmd = watcher['cmd']
            watcher['name'] = watcher.get('name',
                                          os.path.basename(cmd.split()[0]))
            watcher['stream_backend'] = stream_backend
            _watchers.append(Watcher.load_from_config(watcher))

        return Arbiter(_watchers,
                       controller,
                       pubsub_endpoint,
                       statsd=statsd,
                       stats_endpoint=stats_endpoint,
                       statsd_close_outputs=statsd_close_outputs,
                       multicast_endpoint=multicast_endpoint,
                       context=context,
                       plugins=plugins,
                       debug=debug,
                       proc_name=proc_name)
示例#2
0
 def test_start_watcher(self):
     watcher = MockWatcher(name='foo', cmd='serve', priority=1)
     arbiter = Arbiter([], None, None)
     arbiter.start_watcher(watcher)
     self.assertTrue(watcher.started)
示例#3
0
 def test_start_watchers_with_autostart(self):
     watcher = MockWatcher(name='foo', cmd='serve', priority=1,
                           autostart=False)
     arbiter = Arbiter([], None, None)
     arbiter.start_watcher(watcher)
     self.assertFalse(getattr(watcher, 'started', False))
示例#4
0
def get_arbiter(watchers,
                controller=None,
                pubsub_endpoint=None,
                stats_endpoint=None,
                env=None,
                name=None,
                context=None,
                background=False,
                stream_backend="thread",
                plugins=None,
                debug=False,
                proc_name="circusd"):
    """Creates a Arbiter and a single watcher in it.

    Options:

    - **watchers** -- a list of watchers. A watcher in that case is a
      dict containing:

        - **name** -- the name of the watcher (default: None)
        - **cmd** -- the command line used to run the Watcher.
        - **args** -- the args for the command (list or string).
        - **executable** -- When executable is given, the first item in
          the args sequence obtained from **cmd** is still treated by most
          programs as the command name, which can then be different from the
          actual executable name. It becomes the display name for the executing
          program in utilities such as **ps**.

        - **numprocesses** -- the number of processes to spawn (default: 1).
        - **warmup_delay** -- the delay in seconds between two spawns
          (default: 0)
        - **shell** -- if True, the processes are run in the shell
          (default: False)
        - **working_dir** - the working dir for the processes (default: None)
        - **uid** -- the user id used to run the processes (default: None)
        - **gid** -- the group id used to run the processes (default: None)
        - **env** -- the environment passed to the processes (default: None)
        - **send_hup**: if True, a process reload will be done by sending
          the SIGHUP signal. (default: False)
        - **stdout_stream**: a mapping containing the options for configuring
          the stdout stream. Default to None. When provided, may contain:

            - **class**: the fully qualified name of the class to use for
              streaming. Defaults to circus.stream.FileStream
            - **refresh_time**: the delay between two stream checks. Defaults
              to 0.3 seconds.
            - any other key will be passed the class constructor.
        - **stderr_stream**: a mapping containing the options for configuring
          the stderr stream. Default to None. When provided, may contain:

            - **class**: the fully qualified name of the class to use for
              streaming. Defaults to circus.stream.FileStream
            - **refresh_time**: the delay between two stream checks. Defaults
              to 0.3 seconds.
            - any other key will be passed the class constructor.
        - **max_retry**: the number of times we attempt to start a process,
          before we abandon and stop the whole watcher. (default: 5)

    - **controller** -- the zmq entry point (default: 'tcp://127.0.0.1:5555')
    - **pubsub_endpoint** -- the zmq entry point for the pubsub
      (default: 'tcp://127.0.0.1:5556')
    - **stats_endpoint** -- the stats endpoint. If not provided,
      the *circusd-stats* process will not be launched. (default: None)
    - **context** -- the zmq context (default: None)
    - **background** -- If True, the arbiter is launched in a thread in the
      background (default: False)
    - **stream_backend** -- the backend that will be used for the streaming
      process. Can be *thread* or *gevent*. When set to *gevent* you need
      to have *gevent* and *gevent_zmq* installed. (default: thread)
    - **plugins** -- a list of plugins. Each item is a mapping with:

        - **use** -- Fully qualified name that points to the plugin class
        - every other value is passed to the plugin in the **config** option

    - **debug** -- If True the arbiter is launched in debug mode
      (default: False)
    - **proc_name** -- the arbiter process name (default: circusd)
    """
    from circus.util import DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB
    if controller is None:
        controller = DEFAULT_ENDPOINT_DEALER
    if pubsub_endpoint is None:
        pubsub_endpoint = DEFAULT_ENDPOINT_SUB

    if stream_backend == 'gevent':

        try:
            import gevent  # NOQA
            from gevent import monkey  # NOQA
            try:
                import zmq.eventloop as old_io
                import zmq.green as zmq  # NOQA
                old_io.ioloop.Poller = zmq.Poller
            except ImportError:
                # older version
                try:
                    from gevent_zeromq import (  # NOQA
                        monkey_patch, IOLOOP_IS_MONKEYPATCHED)  # NOQA
                    monkey_patch()
                    warnings.warn("gevent_zeromq is deprecated, please "
                                  "use PyZMQ >= 2.2.0.1")
                except ImportError:
                    raise ImportError(_MSG)

            monkey.patch_all()
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

    from circus.watcher import Watcher
    if background:
        from circus.arbiter import ThreadedArbiter as Arbiter  # NOQA
    else:
        from circus.arbiter import Arbiter  # NOQA

    _watchers = []

    for watcher in watchers:
        cmd = watcher['cmd']
        watcher['name'] = watcher.get('name', os.path.basename(cmd.split()[0]))
        watcher['stream_backend'] = stream_backend
        _watchers.append(Watcher.load_from_config(watcher))

    return Arbiter(_watchers,
                   controller,
                   pubsub_endpoint,
                   stats_endpoint=stats_endpoint,
                   context=context,
                   plugins=plugins,
                   debug=debug,
                   proc_name=proc_name)
示例#5
0
def main():
    #configure_logger(logger, 'DEBUG')
    config_mod = 'circus_settings'
    if len(sys.argv) > 1:
        config_mod = sys.argv[1]
    
    config = import_module(config_mod)
    
    
    for p in config.PATHS:
        ap = os.path.abspath(p)
        if ap not in os.sys.path:
            os.sys.path.append(ap)

    watchers = []
    sockets = []

    for s in config.SERVERS:
        
        w = Watcher(gc(s, SERVER_NAME), 
                    gc(s, SERVER_PROGRAM), 
                    gc(s, SERVER_ARGS),
                    numprocesses=gc(s, SERVER_WORKERS, 1), 
                    working_dir=gc(s, SERVER_WORKING_DIR, './'),
                    env=gc(s, SERVER_ENV, dict()),
                    copy_env=True, 
                    copy_path=True,
                    use_sockets=True)
        
        watchers.append(w)
        
        sock_port = gc(s, SERVER_PORT)
        if sock_port is not None:
            sock_name = gc(s, SERVER_NAME)
            sock_host = gc(s, SERVER_HOST, '127.0.0.1')
            sock = CircusSocket(sock_name, host=sock_host, port=sock_port)
            sockets.append(sock)
        
    for sock in sockets:
        print '>> %s'%(sock,)

    try:
        WANT_WEB = getattr(config, 'WANT_WEB')
    except Exception:
        WANT_WEB = True

    if HAS_WEB and WANT_WEB:
        arbiter = Arbiter(watchers, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB, 
                          sockets=sockets,
                        stats_endpoint=DEFAULT_ENDPOINT_STATS, 
                        multicast_endpoint=DEFAULT_ENDPOINT_MULTICAST, 
                        statsd=True, 
                        httpd=True,
                        httpd_port=9999)
    else:
        arbiter = Arbiter(watchers, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
                          sockets=sockets,
                        stats_endpoint=DEFAULT_ENDPOINT_STATS, 
                        multicast_endpoint=DEFAULT_ENDPOINT_MULTICAST)

    arbiter.start()
示例#6
0
 def test_start_watcher(self):
     watcher = MockWatcher(name='foo', cmd='serve', priority=1)
     arbiter = Arbiter([], None, None, check_delay=-1)
     yield arbiter.start_watcher(watcher)
     self.assertTrue(watcher.is_active())
示例#7
0
 def test_ioloop_has_handle_callback_exception(self):
     controller = "tcp://127.0.0.1:%d" % get_available_port()
     sub = "tcp://127.0.0.1:%d" % get_available_port()
     arbiter = Arbiter([], controller, sub, loop=get_ioloop(),
                       check_delay=-1)
     self.assertTrue(hasattr(arbiter.loop, 'handle_callback_exception'))
示例#8
0
def get_arbiter(watchers, controller='tcp://127.0.0.1:5555',
                pubsub_endpoint='tcp://127.0.0.1:5556',
                env=None, name=None, context=None, check_flapping=True,
                background=False, stream_backend="thread"):
    """Creates a Arbiter and a single watcher in it.

    Options:

    - **watchers** -- a list of watchers. A watcher in that case is a
      dict containing:

        - **name** -- the name of the watcher (default: None)
        - **cmd** -- the command line used to run the Watcher.
        - **args** -- the args for the command (list or string).
        - **executable** -- When executable is given, the first item in
          the args sequence obtained from **cmd** is still treated by most
          programs as the command name, which can then be different from the
          actual executable name. It becomes the display name for the executing
          program in utilities such as **ps**.
        - **numprocesses** -- the number of processes to spawn (default: 1).
        - **warmup_delay** -- the delay in seconds between two spawns
          (default: 0)
        - **shell** -- if True, the processes are run in the shell
          (default: False)
        - **working_dir** - the working dir for the processes (default: None)
        - **uid** -- the user id used to run the processes (default: None)
        - **gid** -- the group id used to run the processes (default: None)
        - **env** -- the environment passed to the processes (default: None)
        - **send_hup**: if True, a process reload will be done by sending
          the SIGHUP signal. (default: False)
        - **stdout_stream**: a mapping containing the options for configuring
          the stdout stream. Default to None. When provided, may contain:
            - **class**: the fully qualified name of the class to use for
              streaming. Defaults to circus.stream.FileStream
            - **refresh_time**: the delay between two stream checks. Defaults
              to 0.3 seconds.
            - any other key will be passed the class constructor.
        - **stderr_stream**: a mapping containing the options for configuring
          the stderr stream. Default to None. When provided, may contain:
            - **class**: the fully qualified name of the class to use for
              streaming. Defaults to circus.stream.FileStream
            - **refresh_time**: the delay between two stream checks. Defaults
              to 0.3 seconds.
            - any other key will be passed the class constructor.
         - **flapping_attempts**: number of times a process can restart
           before we start to detect the flapping (default: 2)
        - **flapping_window**: the time window in seconds to test for flapping.
          If the process restarts more than **times** times, we consider it a
          flapping process. (default: 1)
        - **retry_in**: time in seconds to wait until we try to start a process
          that has been flapping. (default: 7)
        - **max_retry**: the number of times we attempt to start a process,
          before we abandon and stop the whole watcher. (default: 5)

    - **controller** -- the zmq entry point (default: 'tcp://127.0.0.1:5555')
    - **pubsub_endpoint** -- the zmq entry point for the pubsub
      (default: 'tcp://127.0.0.1:5556')
    - **context** -- the zmq context (default: None)
    - **check_flapping** -- If True, the flapping detection is activated.
      (default:True)
    - **background** -- If True, the arbiter is launched in a thread in the
      background (default: False)
    - **stream_backend** -- the backend that will be used for the streaming
      process. Can be *thread* or *gevent*. When set to *gevent* you need
      to have *gevent* and *gevent_zmq* installed. (default: thread)
    """
    if stream_backend == 'gevent':
        try:
            import gevent           # NOQA
            import gevent_zeromq    # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

    from circus.watcher import Watcher
    if background:
        from circus.arbiter import ThreadedArbiter as Arbiter   # NOQA
    else:
        from circus.arbiter import Arbiter   # NOQA

    _watchers = []

    for watcher in watchers:
        cmd = watcher['cmd']
        watcher['name'] = watcher.get('name', os.path.basename(cmd.split()[0]))
        watcher['stream_backend'] = stream_backend
        _watchers.append(Watcher.load_from_config(watcher))

    return Arbiter(_watchers, controller, pubsub_endpoint, context=context,
                   check_flapping=check_flapping)