def __call__(self,
watchers,
controller=None,
pubsub_endpoint=None,
statsd=False,
stats_endpoint=None,
statsd_close_outputs=False,
multicast_endpoint=None,
env=None,
name=None,
context=None,
background=False,
stream_backend="thread",
plugins=None,
debug=False,
proc_name="circusd"):
"""Creates a Arbiter and a single watcher in it.
Options:
- **watchers** -- a list of watchers. A watcher in that case is a
dict containing:
- **name** -- the name of the watcher (default: None)
- **cmd** -- the command line used to run the Watcher.
- **args** -- the args for the command (list or string).
- **executable** -- When executable is given, the first item in
the args sequence obtained from **cmd** is still treated by most
programs as the command name, which can then be different from
the actual executable name. It becomes the display name for the
executing program in utilities such as **ps**.
- **numprocesses** -- the number of processes to spawn
(default: 1).
- **warmup_delay** -- the delay in seconds between two spawns
(default: 0)
- **shell** -- if True, the processes are run in the shell
(default: False)
- **working_dir** - the working dir for the processes
(default: None)
- **uid** -- the user id used to run the processes (default: None)
- **gid** -- the group id used to run the processes (default: None)
- **env** -- the environment passed to the processes
(default: None)
- **send_hup**: if True, a process reload will be done by sending
the SIGHUP signal. (default: False)
- **stdout_stream**: a mapping containing the options for
configuring
the stdout stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- any other key will be passed the class constructor.
- **stderr_stream**: a mapping containing the options for
configuring the stderr stream. Default to None. When provided,
may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- any other key will be passed the class constructor.
- **max_retry**: the number of times we attempt to start a process,
before we abandon and stop the whole watcher. (default: 5)
- **hooks**: callback functions for hooking into the watcher
startup and shutdown process. **hooks** is a dict where each key
is the hook name and each value is a 2-tuple with the name of the
callable or the callabled itself and a boolean flag indicating if
an exception occuring in the hook should not be ignored.
Possible values for the hook name: *before_start*, *after_start*,
*before_stop*, *after_stop*, *before_signal*, *after_signal*
- **controller** -- the zmq entry point
(default: 'tcp://127.0.0.1:5555')
- **pubsub_endpoint** -- the zmq entry point for the pubsub
(default: 'tcp://127.0.0.1:5556')
- **stats_endpoint** -- the stats endpoint. If not provided,
the *circusd-stats* process will not be launched. (default: None)
- **statsd_close_outputs** -- if True sends the circusd-stats
stdout/stderr to /dev/null (default: False)
- **context** -- the zmq context (default: None)
- **background** -- If True, the arbiter is launched in a thread in the
background (default: False)
- **stream_backend** -- the backend that will be used for the streaming
process. Can be *thread* or *gevent*. When set to *gevent* you need
to have *gevent* and *gevent_zmq* installed. (default: thread)
- **plugins** -- a list of plugins. Each item is a mapping with:
- **use** -- Fully qualified name that points to the plugin class
- every other value is passed to the plugin in the **config**
option
- **debug** -- If True the arbiter is launched in debug mode
(default: False)
- **proc_name** -- the arbiter process name (default: circusd)
"""
from circus.util import (DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
DEFAULT_ENDPOINT_MULTICAST,
DEFAULT_ENDPOINT_STATS)
if controller is None:
controller = DEFAULT_ENDPOINT_DEALER
if pubsub_endpoint is None:
pubsub_endpoint = DEFAULT_ENDPOINT_SUB
if multicast_endpoint is None:
multicast_endpoint = DEFAULT_ENDPOINT_MULTICAST
if stats_endpoint is None and statsd:
stats_endpoint = DEFAULT_ENDPOINT_STATS
elif stats_endpoint is not None and not statsd:
warnings.warn(
"You defined a stats_endpoint without "
"setting up statsd to True.", DeprecationWarning)
statsd = True
from circus.watcher import Watcher
Arbiter = self._get_arbiter_klass(background=background)
_watchers = []
for watcher in watchers:
cmd = watcher['cmd']
watcher['name'] = watcher.get('name',
os.path.basename(cmd.split()[0]))
watcher['stream_backend'] = stream_backend
_watchers.append(Watcher.load_from_config(watcher))
return Arbiter(_watchers,
controller,
pubsub_endpoint,
statsd=statsd,
stats_endpoint=stats_endpoint,
statsd_close_outputs=statsd_close_outputs,
multicast_endpoint=multicast_endpoint,
context=context,
plugins=plugins,
debug=debug,
proc_name=proc_name)
def test_start_watcher(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1)
arbiter = Arbiter([], None, None)
arbiter.start_watcher(watcher)
self.assertTrue(watcher.started)
def test_start_watchers_with_autostart(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1,
autostart=False)
arbiter = Arbiter([], None, None)
arbiter.start_watcher(watcher)
self.assertFalse(getattr(watcher, 'started', False))
def get_arbiter(watchers,
controller=None,
pubsub_endpoint=None,
stats_endpoint=None,
env=None,
name=None,
context=None,
background=False,
stream_backend="thread",
plugins=None,
debug=False,
proc_name="circusd"):
"""Creates a Arbiter and a single watcher in it.
Options:
- **watchers** -- a list of watchers. A watcher in that case is a
dict containing:
- **name** -- the name of the watcher (default: None)
- **cmd** -- the command line used to run the Watcher.
- **args** -- the args for the command (list or string).
- **executable** -- When executable is given, the first item in
the args sequence obtained from **cmd** is still treated by most
programs as the command name, which can then be different from the
actual executable name. It becomes the display name for the executing
program in utilities such as **ps**.
- **numprocesses** -- the number of processes to spawn (default: 1).
- **warmup_delay** -- the delay in seconds between two spawns
(default: 0)
- **shell** -- if True, the processes are run in the shell
(default: False)
- **working_dir** - the working dir for the processes (default: None)
- **uid** -- the user id used to run the processes (default: None)
- **gid** -- the group id used to run the processes (default: None)
- **env** -- the environment passed to the processes (default: None)
- **send_hup**: if True, a process reload will be done by sending
the SIGHUP signal. (default: False)
- **stdout_stream**: a mapping containing the options for configuring
the stdout stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **stderr_stream**: a mapping containing the options for configuring
the stderr stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **max_retry**: the number of times we attempt to start a process,
before we abandon and stop the whole watcher. (default: 5)
- **controller** -- the zmq entry point (default: 'tcp://127.0.0.1:5555')
- **pubsub_endpoint** -- the zmq entry point for the pubsub
(default: 'tcp://127.0.0.1:5556')
- **stats_endpoint** -- the stats endpoint. If not provided,
the *circusd-stats* process will not be launched. (default: None)
- **context** -- the zmq context (default: None)
- **background** -- If True, the arbiter is launched in a thread in the
background (default: False)
- **stream_backend** -- the backend that will be used for the streaming
process. Can be *thread* or *gevent*. When set to *gevent* you need
to have *gevent* and *gevent_zmq* installed. (default: thread)
- **plugins** -- a list of plugins. Each item is a mapping with:
- **use** -- Fully qualified name that points to the plugin class
- every other value is passed to the plugin in the **config** option
- **debug** -- If True the arbiter is launched in debug mode
(default: False)
- **proc_name** -- the arbiter process name (default: circusd)
"""
from circus.util import DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB
if controller is None:
controller = DEFAULT_ENDPOINT_DEALER
if pubsub_endpoint is None:
pubsub_endpoint = DEFAULT_ENDPOINT_SUB
if stream_backend == 'gevent':
try:
import gevent # NOQA
from gevent import monkey # NOQA
try:
import zmq.eventloop as old_io
import zmq.green as zmq # NOQA
old_io.ioloop.Poller = zmq.Poller
except ImportError:
# older version
try:
from gevent_zeromq import ( # NOQA
monkey_patch, IOLOOP_IS_MONKEYPATCHED) # NOQA
monkey_patch()
warnings.warn("gevent_zeromq is deprecated, please "
"use PyZMQ >= 2.2.0.1")
except ImportError:
raise ImportError(_MSG)
monkey.patch_all()
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from circus.watcher import Watcher
if background:
from circus.arbiter import ThreadedArbiter as Arbiter # NOQA
else:
from circus.arbiter import Arbiter # NOQA
_watchers = []
for watcher in watchers:
cmd = watcher['cmd']
watcher['name'] = watcher.get('name', os.path.basename(cmd.split()[0]))
watcher['stream_backend'] = stream_backend
_watchers.append(Watcher.load_from_config(watcher))
return Arbiter(_watchers,
controller,
pubsub_endpoint,
stats_endpoint=stats_endpoint,
context=context,
plugins=plugins,
debug=debug,
proc_name=proc_name)
def main():
#configure_logger(logger, 'DEBUG')
config_mod = 'circus_settings'
if len(sys.argv) > 1:
config_mod = sys.argv[1]
config = import_module(config_mod)
for p in config.PATHS:
ap = os.path.abspath(p)
if ap not in os.sys.path:
os.sys.path.append(ap)
watchers = []
sockets = []
for s in config.SERVERS:
w = Watcher(gc(s, SERVER_NAME),
gc(s, SERVER_PROGRAM),
gc(s, SERVER_ARGS),
numprocesses=gc(s, SERVER_WORKERS, 1),
working_dir=gc(s, SERVER_WORKING_DIR, './'),
env=gc(s, SERVER_ENV, dict()),
copy_env=True,
copy_path=True,
use_sockets=True)
watchers.append(w)
sock_port = gc(s, SERVER_PORT)
if sock_port is not None:
sock_name = gc(s, SERVER_NAME)
sock_host = gc(s, SERVER_HOST, '127.0.0.1')
sock = CircusSocket(sock_name, host=sock_host, port=sock_port)
sockets.append(sock)
for sock in sockets:
print '>> %s'%(sock,)
try:
WANT_WEB = getattr(config, 'WANT_WEB')
except Exception:
WANT_WEB = True
if HAS_WEB and WANT_WEB:
arbiter = Arbiter(watchers, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
sockets=sockets,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
multicast_endpoint=DEFAULT_ENDPOINT_MULTICAST,
statsd=True,
httpd=True,
httpd_port=9999)
else:
arbiter = Arbiter(watchers, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
sockets=sockets,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
multicast_endpoint=DEFAULT_ENDPOINT_MULTICAST)
arbiter.start()
def test_start_watcher(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1)
arbiter = Arbiter([], None, None, check_delay=-1)
yield arbiter.start_watcher(watcher)
self.assertTrue(watcher.is_active())
def test_ioloop_has_handle_callback_exception(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, loop=get_ioloop(),
check_delay=-1)
self.assertTrue(hasattr(arbiter.loop, 'handle_callback_exception'))
def get_arbiter(watchers, controller='tcp://127.0.0.1:5555',
pubsub_endpoint='tcp://127.0.0.1:5556',
env=None, name=None, context=None, check_flapping=True,
background=False, stream_backend="thread"):
"""Creates a Arbiter and a single watcher in it.
Options:
- **watchers** -- a list of watchers. A watcher in that case is a
dict containing:
- **name** -- the name of the watcher (default: None)
- **cmd** -- the command line used to run the Watcher.
- **args** -- the args for the command (list or string).
- **executable** -- When executable is given, the first item in
the args sequence obtained from **cmd** is still treated by most
programs as the command name, which can then be different from the
actual executable name. It becomes the display name for the executing
program in utilities such as **ps**.
- **numprocesses** -- the number of processes to spawn (default: 1).
- **warmup_delay** -- the delay in seconds between two spawns
(default: 0)
- **shell** -- if True, the processes are run in the shell
(default: False)
- **working_dir** - the working dir for the processes (default: None)
- **uid** -- the user id used to run the processes (default: None)
- **gid** -- the group id used to run the processes (default: None)
- **env** -- the environment passed to the processes (default: None)
- **send_hup**: if True, a process reload will be done by sending
the SIGHUP signal. (default: False)
- **stdout_stream**: a mapping containing the options for configuring
the stdout stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **stderr_stream**: a mapping containing the options for configuring
the stderr stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **flapping_attempts**: number of times a process can restart
before we start to detect the flapping (default: 2)
- **flapping_window**: the time window in seconds to test for flapping.
If the process restarts more than **times** times, we consider it a
flapping process. (default: 1)
- **retry_in**: time in seconds to wait until we try to start a process
that has been flapping. (default: 7)
- **max_retry**: the number of times we attempt to start a process,
before we abandon and stop the whole watcher. (default: 5)
- **controller** -- the zmq entry point (default: 'tcp://127.0.0.1:5555')
- **pubsub_endpoint** -- the zmq entry point for the pubsub
(default: 'tcp://127.0.0.1:5556')
- **context** -- the zmq context (default: None)
- **check_flapping** -- If True, the flapping detection is activated.
(default:True)
- **background** -- If True, the arbiter is launched in a thread in the
background (default: False)
- **stream_backend** -- the backend that will be used for the streaming
process. Can be *thread* or *gevent*. When set to *gevent* you need
to have *gevent* and *gevent_zmq* installed. (default: thread)
"""
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
from circus.watcher import Watcher
if background:
from circus.arbiter import ThreadedArbiter as Arbiter # NOQA
else:
from circus.arbiter import Arbiter # NOQA
_watchers = []
for watcher in watchers:
cmd = watcher['cmd']
watcher['name'] = watcher.get('name', os.path.basename(cmd.split()[0]))
watcher['stream_backend'] = stream_backend
_watchers.append(Watcher.load_from_config(watcher))
return Arbiter(_watchers, controller, pubsub_endpoint, context=context,
check_flapping=check_flapping)