def test_add_watcher_same_lowercase_names(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, loop=get_ioloop(),
check_delay=-1)
arbiter.add_watcher('foo', SLEEP % 5)
self.assertRaises(AlreadyExist, arbiter.add_watcher, 'Foo', SLEEP % 5)
def test_start_watchers_with_autostart(self):
watcher = MockWatcher(name='foo',
cmd='serve',
priority=1,
autostart=False)
arbiter = Arbiter([], None, None, check_delay=-1)
arbiter.start_watcher(watcher)
self.assertFalse(getattr(watcher, 'started', False))
def test_start_arbiter_with_autostart(self):
arbiter = Arbiter([], DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
loop=tornado.ioloop.IOLoop.instance(),
check_delay=-1)
arbiter.add_watcher('foo', 'sleep 5', autostart=False)
try:
yield arbiter.start()
self.assertEqual(arbiter.watchers[0].status(), 'stopped')
finally:
yield arbiter.stop()
def test_start_arbiter_with_autostart(self):
arbiter = Arbiter([], DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
loop=get_ioloop(),
check_delay=-1)
arbiter.add_watcher('foo', SLEEP % 5, autostart=False)
try:
yield arbiter.start()
self.assertEqual(arbiter.watchers[0].status(), 'stopped')
finally:
yield arbiter.stop()
def test_circushttpd(self):
arbiter = Arbiter([], DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
loop=tornado.ioloop.IOLoop.instance(),
check_delay=-1, httpd=True, debug=True)
self.arbiters.append(arbiter)
try:
yield arbiter.start()
poll_for_callable(self.assertDictEqual,
arbiter.statuses, {'circushttpd': 'active'})
finally:
yield arbiter.stop()
def test_add_watcher(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, loop=get_ioloop(),
check_delay=-1)
arbiter.add_watcher('foo', SLEEP % 5)
try:
yield arbiter.start()
self.assertEqual(arbiter.watchers[0].status(), 'active')
finally:
yield arbiter.stop()
def test_circushttpd(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, loop=get_ioloop(),
check_delay=-1, httpd=True, debug=True)
self.arbiters.append(arbiter)
try:
yield arbiter.start()
poll_for_callable(self.assertDictEqual,
arbiter.statuses, {'circushttpd': 'active'})
finally:
yield arbiter.stop()
def test_circushttpd(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, loop=get_ioloop(),
check_delay=-1, httpd=True, debug=True)
self.arbiters.append(arbiter)
try:
yield arbiter.start()
poll_for_callable(self.assertDictEqual,
arbiter.statuses, {'circushttpd': 'active'})
finally:
yield arbiter.stop()
def test_start_watchers_warmup_delay(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1)
arbiter = Arbiter([], None, None, warmup_delay=10)
with patch('circus.arbiter.sleep') as mock_sleep:
arbiter.start_watcher(watcher)
mock_sleep.assert_called_with(10)
# now make sure we don't sleep when there is a autostart
watcher = MockWatcher(name='foo', cmd='serve', priority=1,
autostart=False)
with patch('circus.arbiter.sleep') as mock_sleep:
arbiter.start_watcher(watcher)
assert not mock_sleep.called
def test_start_with_callback(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, check_delay=-1)
callee = mock.MagicMock()
def callback(*args):
callee()
arbiter.stop()
arbiter.start(cb=callback)
self.assertEqual(callee.call_count, 1)
def test_start_with_callback_and_given_loop(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, check_delay=-1,
loop=get_ioloop())
callback = mock.MagicMock()
try:
yield arbiter.start(cb=callback)
finally:
yield arbiter.stop()
self.assertEqual(callback.call_count, 0)
def test_start_with_callback(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, check_delay=-1)
callee = mock.MagicMock()
def callback(*args):
callee()
arbiter.stop()
arbiter.start(cb=callback)
self.assertEqual(callee.call_count, 1)
def test_start_with_callback_and_given_loop(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, check_delay=-1,
loop=get_ioloop())
callback = mock.MagicMock()
try:
yield arbiter.start(cb=callback)
finally:
yield arbiter.stop()
self.assertEqual(callback.call_count, 0)
def test_circushttpd(self):
arbiter = Arbiter([],
DEFAULT_ENDPOINT_DEALER,
DEFAULT_ENDPOINT_SUB,
loop=tornado.ioloop.IOLoop.instance(),
check_delay=-1,
httpd=True,
debug=True)
self.arbiters.append(arbiter)
try:
yield arbiter.start()
poll_for_callable(self.assertDictEqual, arbiter.statuses,
{'circushttpd': 'active'})
finally:
yield arbiter.stop()
def _load_base_arbiter(self):
a = Arbiter.load_from_config(_CONF['reload_base'])
a.evpub_socket = FakeSocket()
# initialize watchers
for watcher in a.iter_watchers():
a._watchers_names[watcher.name.lower()] = watcher
return a
def _load_base_arbiter(self):
a = Arbiter.load_from_config(_CONF['reload_base'])
a.evpub_socket = FakeSocket()
# initialize watchers
for watcher in a.iter_watchers():
a._watchers_names[watcher.name.lower()] = watcher
return a
def setUp(self):
conf = _CONF['reload_base']
self.a = Arbiter.load_from_config(conf)
self.a.evpub_socket = FakeSocket()
# initialize watchers
for watcher in self.a.iter_watchers():
self.a._watchers_names[watcher.name.lower()] = watcher
def _load_base_arbiter(self, name='reload_base'):
loop = tornado.ioloop.IOLoop().instance()
a = Arbiter.load_from_config(_CONF[name], loop=loop)
a.evpub_socket = FakeSocket()
# initialize watchers
for watcher in a.iter_watchers():
a._watchers_names[watcher.name.lower()] = watcher
return a
def setUp(self):
conf = _CONF['reload_base']
self.a = Arbiter.load_from_config(conf)
self.a.evpub_socket = FakeSocket()
# initialize watchers
for watcher in self.a.iter_watchers():
self.a._watchers_names[watcher.name.lower()] = watcher
def _load_base_arbiter(self, name='reload_base'):
loop = tornado.ioloop.IOLoop.current()
a = Arbiter.load_from_config(_CONF[name], loop=loop)
a.evpub_socket = FakeSocket()
# initialize watchers
for watcher in a.iter_watchers():
a._watchers_names[watcher.name.lower()] = watcher
return a
def main():
parser = argparse.ArgumentParser(description='Run some watchers.')
parser.add_argument('config', help='configuration file', nargs='?')
# XXX we should be able to add all these options in the config file as well
parser.add_argument('--log-level',
dest='loglevel',
choices=LOG_LEVELS.keys() +
[key.upper() for key in LOG_LEVELS.keys()],
help="log level")
parser.add_argument(
'--log-output',
dest='logoutput',
help=(
"The location where the logs will be written. The default behavior "
"is to write to stdout (you can force it by passing '-' to "
"this option). Takes a filename otherwise."))
parser.add_argument('--daemon',
dest='daemonize',
action='store_true',
help="Start circusd in the background")
parser.add_argument('--pidfile', dest='pidfile')
parser.add_argument('--version',
action='store_true',
default=False,
help='Displays Circus version and exits.')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.config is None:
parser.print_usage()
sys.exit(0)
if args.daemonize:
daemonize()
# basic logging configuration
logging.basicConfig()
# From here it can also come from the arbiter configuration
# load the arbiter from config
arbiter = Arbiter.load_from_config(args.config)
pidfile = args.pidfile or arbiter.pidfile or None
if pidfile:
pidfile = Pidfile(pidfile)
try:
pidfile.create(os.getpid())
except RuntimeError, e:
print(str(e))
sys.exit(1)
def test_plugin_priority(self):
arbiter = Arbiter.load_from_config(_CONF['issue680'])
watchers = arbiter.iter_watchers()
self.assertEqual(watchers[0].priority, 30)
self.assertEqual(watchers[0].name, 'plugin:myplugin')
self.assertEqual(watchers[1].priority, 20)
self.assertEqual(watchers[1].cmd, 'sleep 20')
self.assertEqual(watchers[2].priority, 10)
self.assertEqual(watchers[2].cmd, 'sleep 10')
def main():
parser = argparse.ArgumentParser(description='Run some watchers.')
parser.add_argument('config', help='configuration file', nargs='?')
# XXX we should be able to add all these options in the config file as well
parser.add_argument('--log-level', dest='loglevel',
choices=LOG_LEVELS.keys() + [key.upper() for key in
LOG_LEVELS.keys()],
help="log level")
parser.add_argument('--log-output', dest='logoutput', help=(
"The location where the logs will be written. The default behavior "
"is to write to stdout (you can force it by passing '-' to "
"this option). Takes a filename otherwise."))
parser.add_argument('--daemon', dest='daemonize', action='store_true',
help="Start circusd in the background")
parser.add_argument('--pidfile', dest='pidfile')
parser.add_argument('--version', action='store_true', default=False,
help='Displays Circus version and exits.')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.config is None:
parser.print_usage()
sys.exit(0)
if args.daemonize:
daemonize()
# basic logging configuration
logging.basicConfig()
# From here it can also come from the arbiter configuration
# load the arbiter from config
arbiter = Arbiter.load_from_config(args.config)
pidfile = args.pidfile or arbiter.pidfile or None
if pidfile:
pidfile = Pidfile(pidfile)
try:
pidfile.create(os.getpid())
except RuntimeError, e:
print(str(e))
sys.exit(1)
def create_standalone_arbiter(
watchers: t.List["Watcher"], **kwargs: t.Any
) -> "Arbiter":
from circus.arbiter import Arbiter # type: ignore[reportMissingTypeStubs]
from . import reserve_free_port
with reserve_free_port() as endpoint_port:
with reserve_free_port() as pubsub_port:
return Arbiter(
watchers,
endpoint=f"tcp://127.0.0.1:{endpoint_port}",
pubsub_endpoint=f"tcp://127.0.0.1:{pubsub_port}",
**kwargs,
)
def test_start_arbiter_with_autostart(self):
arbiter = Arbiter([], DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
loop=get_ioloop(),
check_delay=-1)
arbiter.add_watcher('foo', SLEEP % 5, autostart=False)
try:
yield arbiter.start()
self.assertEqual(arbiter.watchers[0].status(), 'stopped')
finally:
yield arbiter.stop()
def test_add_watcher(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, loop=get_ioloop(),
check_delay=-1)
arbiter.add_watcher('foo', SLEEP % 5)
try:
yield arbiter.start()
self.assertEqual(arbiter.watchers[0].status(), 'active')
finally:
yield arbiter.stop()
def test_add_watcher(self):
arbiter = Arbiter([],
DEFAULT_ENDPOINT_DEALER,
DEFAULT_ENDPOINT_SUB,
loop=tornado.ioloop.IOLoop.instance(),
check_delay=-1)
arbiter.add_watcher('foo', 'sleep 5')
try:
yield arbiter.start()
self.assertEqual(arbiter.watchers[0].status(), 'active')
finally:
yield arbiter.stop()
def test_start_watchers_warmup_delay(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1)
arbiter = Arbiter([], None, None, warmup_delay=10)
with patch('circus.arbiter.sleep') as mock_sleep:
arbiter.start_watcher(watcher)
mock_sleep.assert_called_with(10)
# now make sure we don't sleep when there is a autostart
watcher = MockWatcher(name='foo', cmd='serve', priority=1,
autostart=False)
with patch('circus.arbiter.sleep') as mock_sleep:
arbiter.start_watcher(watcher)
assert not mock_sleep.called
def test_reload(self):
a = Arbiter.load_from_config(_CONF["reload1"])
a.initialize()
self.assertEqual(len(a.watchers), 1)
a.reload_from_config(_CONF["reload2"])
self.assertEqual(len(a.watchers), 2)
a.reload_from_config(_CONF["reload3"])
self.assertEqual(len(a.watchers), 1)
a.reload_from_config(_CONF["reload4"])
self.assertEqual(len(a.watchers), 1)
self.assertEqual(a.watchers[0].name, "test3")
self.assertEqual(a.watchers[0].numprocesses, 1)
w = a.watchers[0]
a.reload_from_config(_CONF["reload5"])
self.assertEqual(a.watchers[0].name, "test3")
self.assertEqual(a.watchers[0].numprocesses, 2)
# check that just the number of processes is changed and that the watcher it self is not changed
self.assertEqual(a.watchers[0], w)
a.evpub_socket.close()
a.stop()
def test_start_watchers_with_autostart(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1,
autostart=False)
arbiter = Arbiter([], None, None)
arbiter.start_watcher(watcher)
self.assertFalse(getattr(watcher, 'started', False))
def test_start_watcher(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1)
arbiter = Arbiter([], None, None)
arbiter.start_watcher(watcher)
self.assertTrue(watcher.started)
def test_start_watcher(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1)
arbiter = Arbiter([], None, None)
arbiter.start_watcher(watcher)
self.assertTrue(watcher.started)
limit = cfg_name[7:]
rlimits[limit] = int(cfg_value)
watcher = Watcher(name, cmd, numprocesses=numprocesses,
warmup_delay=warmup_delay, working_dir=working_dir,
shell=shell, uid=uid, gid=gid, send_hup=send_hup,
times=times, within=within, retry_in=retry_in,
max_retry=max_retry, graceful_timeout=graceful_timeout,
rlimits=rlimits)
watchers.append(watcher)
# main circus options
check = cfg.dget('circus', 'check_delay', 5, int)
endpoint = cfg.dget('circus', 'endpoint', 'tcp://127.0.0.1:5555')
pubsub_endpoint = cfg.dget('circus', 'pubsub_endpoint',
'tcp://127.0.0.1:5556')
arbiter = Arbiter(watchers, endpoint, pubsub_endpoint, check)
try:
arbiter.start()
finally:
arbiter.stop()
if pidfile is not None:
pidfile.unlink()
sys.exit(0)
if __name__ == '__main__':
main()
def main():
import zmq
try:
zmq_version = [int(part) for part in zmq.__version__.split(".")]
if len(zmq_version) < 2:
raise ValueError()
except (AttributeError, ValueError):
print("Unknown PyZQM version - aborting...")
sys.exit(0)
if zmq_version[0] < 13 or (zmq_version[0] == 13 and zmq_version[1] < 1):
print("circusd needs PyZMQ >= 13.1.0 to run - aborting...")
sys.exit(0)
parser = argparse.ArgumentParser(description="Run some watchers.")
parser.add_argument("config", help="configuration file", nargs="?")
# XXX we should be able to add all these options in the config file as well
parser.add_argument(
"--log-level",
dest="loglevel",
choices=list(LOG_LEVELS.keys()) + [key.upper() for key in LOG_LEVELS.keys()],
help="log level",
)
parser.add_argument(
"--log-output",
dest="logoutput",
help=(
"The location where the logs will be written. The default behavior "
"is to write to stdout (you can force it by passing '-' to "
"this option). Takes a filename otherwise."
),
)
parser.add_argument(
"--logger-config",
dest="loggerconfig",
help=(
"The location where a standard Python logger configuration INI, "
"JSON or YAML file can be found. This can be used to override "
"the default logging configuration for the arbiter."
),
)
parser.add_argument("--daemon", dest="daemonize", action="store_true", help="Start circusd in the background")
parser.add_argument("--pidfile", dest="pidfile")
parser.add_argument("--version", action="store_true", default=False, help="Displays Circus version and exits.")
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.config is None:
parser.print_usage()
sys.exit(0)
if args.daemonize:
daemonize()
# From here it can also come from the arbiter configuration
# load the arbiter from config
arbiter = Arbiter.load_from_config(args.config)
# go ahead and set umask early if it is in the config
if arbiter.umask is not None:
os.umask(arbiter.umask)
pidfile = args.pidfile or arbiter.pidfile or None
if pidfile:
pidfile = Pidfile(pidfile)
try:
pidfile.create(os.getpid())
except RuntimeError as e:
print(str(e))
sys.exit(1)
# configure the logger
loglevel = args.loglevel or arbiter.loglevel or "info"
logoutput = args.logoutput or arbiter.logoutput or "-"
loggerconfig = args.loggerconfig or arbiter.loggerconfig or None
configure_logger(logger, loglevel, logoutput, loggerconfig)
# Main loop
restart = True
while restart:
try:
arbiter = arbiter or Arbiter.load_from_config(args.config)
future = arbiter.start()
restart = False
if check_future_exception_and_log(future) is None:
restart = arbiter._restarting
except Exception as e:
# emergency stop
arbiter.loop.run_sync(arbiter._emergency_stop)
raise (e)
except KeyboardInterrupt:
pass
finally:
arbiter = None
if pidfile is not None:
pidfile.unlink()
sys.exit(0)
def test_ioloop_has_handle_callback_exception(self):
controller = "tcp://127.0.0.1:%d" % get_available_port()
sub = "tcp://127.0.0.1:%d" % get_available_port()
arbiter = Arbiter([], controller, sub, loop=get_ioloop(),
check_delay=-1)
self.assertTrue(hasattr(arbiter.loop, 'handle_callback_exception'))
def main():
#configure_logger(logger, 'DEBUG')
config_mod = 'circus_settings'
if len(sys.argv) > 1:
config_mod = sys.argv[1]
config = import_module(config_mod)
for p in config.PATHS:
ap = os.path.abspath(p)
if ap not in os.sys.path:
os.sys.path.append(ap)
watchers = []
sockets = []
for s in config.SERVERS:
w = Watcher(gc(s, SERVER_NAME),
gc(s, SERVER_PROGRAM),
gc(s, SERVER_ARGS),
numprocesses=gc(s, SERVER_WORKERS, 1),
working_dir=gc(s, SERVER_WORKING_DIR, './'),
env=gc(s, SERVER_ENV, dict()),
copy_env=True,
copy_path=True,
use_sockets=True)
watchers.append(w)
sock_port = gc(s, SERVER_PORT)
if sock_port is not None:
sock_name = gc(s, SERVER_NAME)
sock_host = gc(s, SERVER_HOST, '127.0.0.1')
sock = CircusSocket(sock_name, host=sock_host, port=sock_port)
sockets.append(sock)
for sock in sockets:
print '>> %s'%(sock,)
try:
WANT_WEB = getattr(config, 'WANT_WEB')
except Exception:
WANT_WEB = True
if HAS_WEB and WANT_WEB:
arbiter = Arbiter(watchers, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
sockets=sockets,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
multicast_endpoint=DEFAULT_ENDPOINT_MULTICAST,
statsd=True,
httpd=True,
httpd_port=9999)
else:
arbiter = Arbiter(watchers, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
sockets=sockets,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
multicast_endpoint=DEFAULT_ENDPOINT_MULTICAST)
arbiter.start()
def test_start_watcher(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1)
arbiter = Arbiter([], None, None, check_delay=-1)
yield arbiter.start_watcher(watcher)
self.assertTrue(watcher.is_active())
def test_start_watcher(self):
watcher = MockWatcher(name='foo', cmd='serve', priority=1)
arbiter = Arbiter([], None, None, check_delay=-1)
yield arbiter.start_watcher(watcher)
self.assertTrue(watcher.is_active())
def main():
import zmq
try:
zmq_version = [int(part) for part in zmq.__version__.split('.')]
if len(zmq_version) < 2:
raise ValueError()
except (AttributeError, ValueError):
print('Unknown PyZQM version - aborting...')
sys.exit(0)
if zmq_version[0] < 13 or (zmq_version[0] == 13 and zmq_version[1] < 1):
print('circusd needs PyZMQ >= 13.1.0 to run - aborting...')
sys.exit(0)
parser = argparse.ArgumentParser(description='Run some watchers.')
parser.add_argument('config', help='configuration file', nargs='?')
# XXX we should be able to add all these options in the config file as well
parser.add_argument('--log-level',
dest='loglevel',
choices=list(LOG_LEVELS.keys()) +
[key.upper() for key in LOG_LEVELS.keys()],
help="log level")
parser.add_argument(
'--log-output',
dest='logoutput',
help=(
"The location where the logs will be written. The default behavior "
"is to write to stdout (you can force it by passing '-' to "
"this option). Takes a filename otherwise."))
parser.add_argument(
"--logger-config",
dest="loggerconfig",
help=("The location where a standard Python logger configuration INI, "
"JSON or YAML file can be found. This can be used to override "
"the default logging configuration for the arbiter."))
parser.add_argument('--daemon',
dest='daemonize',
action='store_true',
help="Start circusd in the background")
parser.add_argument('--pidfile', dest='pidfile')
parser.add_argument('--version',
action='store_true',
default=False,
help='Displays Circus version and exits.')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.config is None:
parser.print_usage()
sys.exit(0)
if args.daemonize:
daemonize()
# From here it can also come from the arbiter configuration
# load the arbiter from config
arbiter = Arbiter.load_from_config(args.config)
# go ahead and set umask early if it is in the config
if arbiter.umask is not None:
os.umask(arbiter.umask)
pidfile = args.pidfile or arbiter.pidfile or None
if pidfile:
pidfile = Pidfile(pidfile)
try:
pidfile.create(os.getpid())
except RuntimeError as e:
print(str(e))
sys.exit(1)
# configure the logger
loglevel = args.loglevel or arbiter.loglevel or 'info'
logoutput = args.logoutput or arbiter.logoutput or '-'
loggerconfig = args.loggerconfig or arbiter.loggerconfig or None
configure_logger(logger, loglevel, logoutput, loggerconfig)
# Main loop
restart = True
while restart:
try:
arbiter = arbiter or Arbiter.load_from_config(args.config)
future = arbiter.start()
restart = False
if check_future_exception_and_log(future) is None:
restart = arbiter._restarting
except Exception as e:
# emergency stop
arbiter.loop.run_sync(arbiter._emergency_stop)
raise (e)
except KeyboardInterrupt:
pass
finally:
arbiter = None
if pidfile is not None:
pidfile.unlink()
sys.exit(0)
def main():
parser = argparse.ArgumentParser(description='Run some watchers.')
parser.add_argument('config', help='configuration file', nargs='?')
# XXX we should be able to add all these options in the config file as well
parser.add_argument('--log-level',
dest='loglevel',
choices=list(LOG_LEVELS.keys()) +
[key.upper() for key in LOG_LEVELS.keys()],
help="log level")
parser.add_argument(
'--log-output',
dest='logoutput',
help=(
"The location where the logs will be written. The default behavior "
"is to write to stdout (you can force it by passing '-' to "
"this option). Takes a filename otherwise."))
parser.add_argument('--daemon',
dest='daemonize',
action='store_true',
help="Start circusd in the background")
parser.add_argument('--pidfile', dest='pidfile')
parser.add_argument('--version',
action='store_true',
default=False,
help='Displays Circus version and exits.')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.config is None:
parser.print_usage()
sys.exit(0)
if args.daemonize:
daemonize()
# basic logging configuration
logging.basicConfig()
# From here it can also come from the arbiter configuration
# load the arbiter from config
arbiter = Arbiter.load_from_config(args.config)
pidfile = args.pidfile or arbiter.pidfile or None
if pidfile:
pidfile = Pidfile(pidfile)
try:
pidfile.create(os.getpid())
except RuntimeError as e:
print(str(e))
sys.exit(1)
# configure the logger
loglevel = args.loglevel or arbiter.loglevel or 'info'
logoutput = args.logoutput or arbiter.logoutput or '-'
configure_logger(logger, loglevel, logoutput)
# configure the main loop
#ioloop.install()
#loop = ioloop.IOLoop.instance()
#cb = functools.partial(manage_restart, loop, arbiter)
#periodic = tornado.ioloop.PeriodicCallback(cb, 1000, loop)
#periodic.start()
# schedule the arbiter start
#arbiter = Arbiter.load_from_config(args.config, loop=loop)
#loop.add_future(arbiter.start(), _arbiter_start_cb)
# Main loop
restart = True
while restart:
try:
arbiter = Arbiter.load_from_config(args.config)
future = arbiter.start()
restart = False
if check_future_exception_and_log(future) is None:
restart = arbiter._restarting
except Exception as e:
# emergency stop
arbiter.loop.run_sync(arbiter._emergency_stop)
raise (e)
except KeyboardInterrupt:
pass
finally:
if pidfile is not None:
pidfile.unlink()
sys.exit(0)
pidfile.create(os.getpid())
except RuntimeError, e:
print(str(e))
sys.exit(1)
# configure the logger
loglevel = args.loglevel or arbiter.loglevel or 'info'
logoutput = args.logoutput or arbiter.logoutput or '-'
configure_logger(logger, loglevel, logoutput)
try:
restart_after_stop = True
while restart_after_stop:
while True:
try:
arbiter = Arbiter.load_from_config(args.config)
restart_after_stop = arbiter.start()
except ReloadArbiterException:
pass
else:
break
except KeyboardInterrupt:
pass
finally:
arbiter.stop()
if pidfile is not None:
pidfile.unlink()
sys.exit(0)
def main():
import zmq
try:
zmq_version = [int(part) for part in zmq.__version__.split('.')]
if len(zmq_version) < 2:
raise ValueError()
except (AttributeError, ValueError):
print('Unknown PyZQM version - aborting...')
sys.exit(0)
if zmq_version[0] < 13 or (zmq_version[0] == 13 and zmq_version[1] < 1):
print('circusd needs PyZMQ >= 13.1.0 to run - aborting...')
sys.exit(0)
parser = argparse.ArgumentParser(description='Run some watchers.')
parser.add_argument('config', help='configuration file', nargs='?')
# XXX we should be able to add all these options in the config file as well
parser.add_argument('--log-level', dest='loglevel',
choices=list(LOG_LEVELS.keys()) + [
key.upper() for key in LOG_LEVELS.keys()],
help="log level")
parser.add_argument('--log-output', dest='logoutput', help=(
"The location where the logs will be written. The default behavior "
"is to write to stdout (you can force it by passing '-' to "
"this option). Takes a filename otherwise."))
parser.add_argument('--daemon', dest='daemonize', action='store_true',
help="Start circusd in the background")
parser.add_argument('--pidfile', dest='pidfile')
parser.add_argument('--version', action='store_true', default=False,
help='Displays Circus version and exits.')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.config is None:
parser.print_usage()
sys.exit(0)
if args.daemonize:
daemonize()
# This config call is done to avoid any
# "no handlers could be found for logger"
#
# error while loding the configuration and setting up the arbiter.
# The real logging configuration is done right after via
# a configure_logger() call
logging.basicConfig()
# From here it can also come from the arbiter configuration
# load the arbiter from config
arbiter = Arbiter.load_from_config(args.config)
# go ahead and set umask early if it is in the config
if arbiter.umask is not None:
os.umask(arbiter.umask)
pidfile = args.pidfile or arbiter.pidfile or None
if pidfile:
pidfile = Pidfile(pidfile)
try:
pidfile.create(os.getpid())
except RuntimeError as e:
print(str(e))
sys.exit(1)
# configure the logger
loglevel = args.loglevel or arbiter.loglevel or 'info'
logoutput = args.logoutput or arbiter.logoutput or '-'
configure_logger(logger, loglevel, logoutput)
# Main loop
restart = True
while restart:
try:
arbiter = arbiter or Arbiter.load_from_config(args.config)
future = arbiter.start()
restart = False
if check_future_exception_and_log(future) is None:
restart = arbiter._restarting
except Exception as e:
# emergency stop
arbiter.loop.run_sync(arbiter._emergency_stop)
raise(e)
except KeyboardInterrupt:
pass
finally:
arbiter = None
if pidfile is not None:
pidfile.unlink()
sys.exit(0)
def get_arbiter(watchers,
controller='tcp://127.0.0.1:5555',
pubsub_endpoint='tcp://127.0.0.1:5556',
stats_endpoint=None,
env=None,
name=None,
context=None,
background=False,
stream_backend="thread",
plugins=None):
"""Creates a Arbiter and a single watcher in it.
Options:
- **watchers** -- a list of watchers. A watcher in that case is a
dict containing:
- **name** -- the name of the watcher (default: None)
- **cmd** -- the command line used to run the Watcher.
- **args** -- the args for the command (list or string).
- **executable** -- When executable is given, the first item in
the args sequence obtained from **cmd** is still treated by most
programs as the command name, which can then be different from the
actual executable name. It becomes the display name for the executing
program in utilities such as **ps**.
- **numprocesses** -- the number of processes to spawn (default: 1).
- **warmup_delay** -- the delay in seconds between two spawns
(default: 0)
- **shell** -- if True, the processes are run in the shell
(default: False)
- **working_dir** - the working dir for the processes (default: None)
- **uid** -- the user id used to run the processes (default: None)
- **gid** -- the group id used to run the processes (default: None)
- **env** -- the environment passed to the processes (default: None)
- **send_hup**: if True, a process reload will be done by sending
the SIGHUP signal. (default: False)
- **stdout_stream**: a mapping containing the options for configuring
the stdout stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **stderr_stream**: a mapping containing the options for configuring
the stderr stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **max_retry**: the number of times we attempt to start a process,
before we abandon and stop the whole watcher. (default: 5)
- **controller** -- the zmq entry point (default: 'tcp://127.0.0.1:5555')
- **pubsub_endpoint** -- the zmq entry point for the pubsub
(default: 'tcp://127.0.0.1:5556')
- **stats_endpoint** -- the stats endpoint. If not provided,
the *circusd-stats* process will not be launched. (default: None)
- **context** -- the zmq context (default: None)
- **background** -- If True, the arbiter is launched in a thread in the
background (default: False)
- **stream_backend** -- the backend that will be used for the streaming
process. Can be *thread* or *gevent*. When set to *gevent* you need
to have *gevent* and *gevent_zmq* installed. (default: thread)
- **plugins** -- a list of plugins. Each item is a mapping with:
- **use** -- Fully qualified name that points to the plugin class
- every other value is passed to the plugin in the **config** option
"""
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
from circus.watcher import Watcher
if background:
from circus.arbiter import ThreadedArbiter as Arbiter # NOQA
else:
from circus.arbiter import Arbiter # NOQA
_watchers = []
for watcher in watchers:
cmd = watcher['cmd']
watcher['name'] = watcher.get('name', os.path.basename(cmd.split()[0]))
watcher['stream_backend'] = stream_backend
_watchers.append(Watcher.load_from_config(watcher))
return Arbiter(_watchers,
controller,
pubsub_endpoint,
stats_endpoint=stats_endpoint,
context=context,
plugins=plugins)
def test_find_hook_in_pythonpath(self):
arbiter = Arbiter.load_from_config(_CONF['find_hook_in_pythonpath'])
watcher = arbiter.iter_watchers()[0]
self.assertEqual(watcher.hooks['before_start'].__doc__,
'relative_hook')
self.assertTrue('before_start' not in watcher.ignore_hook_failure)
def __call__(self,
watchers,
controller=None,
pubsub_endpoint=None,
statsd=False,
stats_endpoint=None,
statsd_close_outputs=False,
multicast_endpoint=None,
env=None,
name=None,
context=None,
background=False,
stream_backend="thread",
plugins=None,
debug=False,
proc_name="circusd"):
"""Creates a Arbiter and a single watcher in it.
Options:
- **watchers** -- a list of watchers. A watcher in that case is a
dict containing:
- **name** -- the name of the watcher (default: None)
- **cmd** -- the command line used to run the Watcher.
- **args** -- the args for the command (list or string).
- **executable** -- When executable is given, the first item in
the args sequence obtained from **cmd** is still treated by most
programs as the command name, which can then be different from
the actual executable name. It becomes the display name for the
executing program in utilities such as **ps**.
- **numprocesses** -- the number of processes to spawn
(default: 1).
- **warmup_delay** -- the delay in seconds between two spawns
(default: 0)
- **shell** -- if True, the processes are run in the shell
(default: False)
- **working_dir** - the working dir for the processes
(default: None)
- **uid** -- the user id used to run the processes (default: None)
- **gid** -- the group id used to run the processes (default: None)
- **env** -- the environment passed to the processes
(default: None)
- **send_hup**: if True, a process reload will be done by sending
the SIGHUP signal. (default: False)
- **stdout_stream**: a mapping containing the options for
configuring
the stdout stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- any other key will be passed the class constructor.
- **stderr_stream**: a mapping containing the options for
configuring the stderr stream. Default to None. When provided,
may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- any other key will be passed the class constructor.
- **max_retry**: the number of times we attempt to start a process,
before we abandon and stop the whole watcher. (default: 5)
- **hooks**: callback functions for hooking into the watcher
startup and shutdown process. **hooks** is a dict where each key
is the hook name and each value is a 2-tuple with the name of the
callable or the callabled itself and a boolean flag indicating if
an exception occuring in the hook should not be ignored.
Possible values for the hook name: *before_start*, *after_start*,
*before_stop*, *after_stop*, *before_signal*, *after_signal*
- **controller** -- the zmq entry point
(default: 'tcp://127.0.0.1:5555')
- **pubsub_endpoint** -- the zmq entry point for the pubsub
(default: 'tcp://127.0.0.1:5556')
- **stats_endpoint** -- the stats endpoint. If not provided,
the *circusd-stats* process will not be launched. (default: None)
- **statsd_close_outputs** -- if True sends the circusd-stats
stdout/stderr to /dev/null (default: False)
- **context** -- the zmq context (default: None)
- **background** -- If True, the arbiter is launched in a thread in the
background (default: False)
- **stream_backend** -- the backend that will be used for the streaming
process. Can be *thread* or *gevent*. When set to *gevent* you need
to have *gevent* and *gevent_zmq* installed. (default: thread)
- **plugins** -- a list of plugins. Each item is a mapping with:
- **use** -- Fully qualified name that points to the plugin class
- every other value is passed to the plugin in the **config**
option
- **debug** -- If True the arbiter is launched in debug mode
(default: False)
- **proc_name** -- the arbiter process name (default: circusd)
"""
from circus.util import (DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
DEFAULT_ENDPOINT_MULTICAST,
DEFAULT_ENDPOINT_STATS)
if controller is None:
controller = DEFAULT_ENDPOINT_DEALER
if pubsub_endpoint is None:
pubsub_endpoint = DEFAULT_ENDPOINT_SUB
if multicast_endpoint is None:
multicast_endpoint = DEFAULT_ENDPOINT_MULTICAST
if stats_endpoint is None and statsd:
stats_endpoint = DEFAULT_ENDPOINT_STATS
elif stats_endpoint is not None and not statsd:
warnings.warn(
"You defined a stats_endpoint without "
"setting up statsd to True.", DeprecationWarning)
statsd = True
from circus.watcher import Watcher
Arbiter = self._get_arbiter_klass(background=background)
_watchers = []
for watcher in watchers:
cmd = watcher['cmd']
watcher['name'] = watcher.get('name',
os.path.basename(cmd.split()[0]))
watcher['stream_backend'] = stream_backend
_watchers.append(Watcher.load_from_config(watcher))
return Arbiter(_watchers,
controller,
pubsub_endpoint,
statsd=statsd,
stats_endpoint=stats_endpoint,
statsd_close_outputs=statsd_close_outputs,
multicast_endpoint=multicast_endpoint,
context=context,
plugins=plugins,
debug=debug,
proc_name=proc_name)
def main():
parser = argparse.ArgumentParser(description='Run some watchers.')
parser.add_argument('config', help='configuration file', nargs='?')
# XXX we should be able to add all these options in the config file as well
parser.add_argument('--log-level', dest='loglevel',
choices=list(LOG_LEVELS.keys()) + [
key.upper() for key in LOG_LEVELS.keys()],
help="log level")
parser.add_argument('--log-output', dest='logoutput', help=(
"The location where the logs will be written. The default behavior "
"is to write to stdout (you can force it by passing '-' to "
"this option). Takes a filename otherwise."))
parser.add_argument('--daemon', dest='daemonize', action='store_true',
help="Start circusd in the background")
parser.add_argument('--pidfile', dest='pidfile')
parser.add_argument('--version', action='store_true', default=False,
help='Displays Circus version and exits.')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.config is None:
parser.print_usage()
sys.exit(0)
if args.daemonize:
daemonize()
# basic logging configuration
logging.basicConfig()
# From here it can also come from the arbiter configuration
# load the arbiter from config
arbiter = Arbiter.load_from_config(args.config)
pidfile = args.pidfile or arbiter.pidfile or None
if pidfile:
pidfile = Pidfile(pidfile)
try:
pidfile.create(os.getpid())
except RuntimeError as e:
print(str(e))
sys.exit(1)
# configure the logger
loglevel = args.loglevel or arbiter.loglevel or 'info'
logoutput = args.logoutput or arbiter.logoutput or '-'
configure_logger(logger, loglevel, logoutput)
# configure the main loop
#ioloop.install()
#loop = ioloop.IOLoop.instance()
#cb = functools.partial(manage_restart, loop, arbiter)
#periodic = tornado.ioloop.PeriodicCallback(cb, 1000, loop)
#periodic.start()
# schedule the arbiter start
#arbiter = Arbiter.load_from_config(args.config, loop=loop)
#loop.add_future(arbiter.start(), _arbiter_start_cb)
# Main loop
restart = True
while restart:
try:
arbiter = Arbiter.load_from_config(args.config)
future = arbiter.start()
restart = False
if check_future_exception_and_log(future) is None:
restart = arbiter._restarting
except Exception as e:
# emergency stop
arbiter.loop.run_sync(arbiter._emergency_stop)
raise(e)
except KeyboardInterrupt:
pass
finally:
if pidfile is not None:
pidfile.unlink()
sys.exit(0)
pidfile = None
if args.pidfile:
pidfile = Pidfile(args.pidfile)
try:
pidfile.create(os.getpid())
except RuntimeError, e:
print(str(e))
sys.exit(1)
# configure the logger
configure_logger(logger, args.loglevel, args.logoutput)
# load the arbiter from config
arbiter = Arbiter.load_from_config(args.config)
try:
arbiter.start()
except KeyboardInterrupt:
pass
finally:
arbiter.stop()
if pidfile is not None:
pidfile.unlink()
sys.exit(0)
if __name__ == '__main__':
main()
def test_find_hook_in_pythonpath(self):
arbiter = Arbiter.load_from_config(_CONF['find_hook_in_pythonpath'])
watcher = arbiter.iter_watchers()[0]
self.assertEqual(watcher.hooks['before_start'].__doc__,
'relative_hook')
self.assertTrue('before_start' not in watcher.ignore_hook_failure)