def test_socket(self):
if TRAVIS:
return
sock = CircusSocket('somename', 'localhost', 0)
try:
sock.bind_and_listen()
finally:
sock.close()
def test_unix_socket(self):
sockfile = self._get_tmp_filename()
sock = CircusSocket('somename', path=sockfile, umask=0)
try:
sock.bind_and_listen()
self.assertTrue(os.path.exists(sockfile))
permissions = oct(os.stat(sockfile).st_mode)[-3:]
self.assertEqual(permissions, '777')
finally:
sock.close()
def test_unix_socket(self):
fd, sockfile = tempfile.mkstemp()
os.close(fd)
os.remove(sockfile)
sock = CircusSocket('somename', path=sockfile)
try:
sock.bind_and_listen()
self.assertTrue(os.path.exists(sockfile))
finally:
sock.close()
os.remove(sockfile)
def test_unix_socket(self):
fd, sockfile = tempfile.mkstemp()
os.close(fd)
os.remove(sockfile)
sock = CircusSocket('somename', path=sockfile, umask=0)
try:
sock.bind_and_listen()
self.assertTrue(os.path.exists(sockfile))
permissions = oct(os.stat(sockfile).st_mode)[-3:]
self.assertEqual(permissions, '777')
finally:
sock.close()
def load_from_config(cls, config_file):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
ssh_server=cfg.get('ssh_server', None))
return arbiter
def _get_sockets_fds(self):
"""Returns sockets dict. If this worker's cmd indicates use of
a SO_REUSEPORT socket, a new socket is created and bound. This
new socket's FD replaces original socket's FD in returned dict.
This method populates `self._sockets` list. This list should be
let go after `fork()`.
"""
sockets_fds = None
if self.watcher is not None and self.watcher.sockets is not None:
sockets_fds = self.watcher._get_sockets_fds()
reuseport_sockets = tuple((sn, s) for (sn, s)
in self.watcher.sockets.items()
if s.so_reuseport)
for sn, s in reuseport_sockets:
# watcher.cmd uses this reuseport socket
if 'circus.sockets.%s' % sn in self.watcher.cmd:
sock = CircusSocket.load_from_config(s._cfg)
sock.bind_and_listen()
# replace original socket's fd
sockets_fds[sn] = sock.fileno()
# keep new socket until fork returns
self._sockets.append(sock)
return sockets_fds
def load_from_config(cls, config_file):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=cfg.get('httpd', False),
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
stream_backend=cfg.get('stream_backend', 'thread'),
ssh_server=cfg.get('ssh_server', None))
return arbiter
def load_from_config(cls, config_file):
cfg = get_config(config_file)
# hack reload ioloop to use the monkey patched version
reload(ioloop)
watchers = []
for watcher in cfg.get("watchers", []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get("sockets", []):
sockets.append(CircusSocket.load_from_config(socket))
# creating arbiter
arbiter = cls(
watchers,
cfg["endpoint"],
cfg["pubsub_endpoint"],
check_delay=cfg.get("check_delay", 1.0),
prereload_fn=cfg.get("prereload_fn"),
stats_endpoint=cfg.get("stats_endpoint"),
plugins=cfg.get("plugins"),
sockets=sockets,
)
return arbiter
def test_issue310(self):
"""
https://github.com/mozilla-services/circus/pull/310
Allow $(circus.sockets.name) to be used in args.
"""
conf = get_config(_CONF["issue310"])
watcher = Watcher.load_from_config(conf["watchers"][0])
socket = CircusSocket.load_from_config(conf["sockets"][0])
watcher.initialize(None, {"web": socket}, None)
process = Process(
watcher._process_counter,
watcher.cmd,
args=watcher.args,
working_dir=watcher.working_dir,
shell=watcher.shell,
uid=watcher.uid,
gid=watcher.gid,
env=watcher.env,
rlimits=watcher.rlimits,
spawn=False,
executable=watcher.executable,
use_fds=watcher.use_sockets,
watcher=watcher,
)
fd = watcher._get_sockets_fds()["web"]
formatted_args = process.format_args()
self.assertEquals(formatted_args, ["foo", "--fd", str(fd)])
def test_issue310(self):
'''
https://github.com/mozilla-services/circus/pull/310
Allow $(circus.sockets.name) to be used in args.
'''
conf = get_config(_CONF['issue310'])
watcher = Watcher.load_from_config(conf['watchers'][0])
socket = CircusSocket.load_from_config(conf['sockets'][0])
try:
watcher.initialize(None, {'web': socket}, None)
process = Process(watcher._nextwid, watcher.cmd,
args=watcher.args,
working_dir=watcher.working_dir,
shell=watcher.shell, uid=watcher.uid,
gid=watcher.gid, env=watcher.env,
rlimits=watcher.rlimits, spawn=False,
executable=watcher.executable,
use_fds=watcher.use_sockets,
watcher=watcher)
sockets_fds = watcher._get_sockets_fds()
formatted_args = process.format_args(sockets_fds=sockets_fds)
fd = sockets_fds['web']
self.assertEqual(formatted_args,
['foo', '--fd', str(fd)])
finally:
socket.close()
def load_from_config(cls, config_file):
cfg = get_config(config_file)
# hack reload ioloop to use the monkey patched version
reload(ioloop)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=cfg.get('httpd', False),
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080))
return arbiter
def test_reuseport_supported(self):
config = {"name": "", "host": "localhost", "port": 0, "so_reuseport": True}
sock = CircusSocket.load_from_config(config)
sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT)
self.assertEqual(sock.so_reuseport, True)
self.assertNotEqual(sockopt, 0)
def test_load_from_config_replace(self):
sockfile = self._get_file()
config = {'name': 'somename', 'path': sockfile, 'replace': False}
sock = CircusSocket.load_from_config(config)
try:
self.assertRaises(OSError, sock.bind_and_listen)
finally:
sock.close()
config = {'name': 'somename', 'path': sockfile, 'replace': True}
sock = CircusSocket.load_from_config(config)
sock.bind_and_listen()
try:
self.assertEqual(sock.replace, True)
finally:
sock.close()
def test_load_from_config_umask(self):
sockfile = self._get_tmp_filename()
config = {'name': 'somename', 'path': sockfile, 'umask': 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def test_load_from_config_umask(self):
sockfile = self._get_tmp_filename()
config = {"name": "somename", "path": sockfile, "umask": 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def test_load_from_config_blocking(self):
# test default to false
config = {'name': 'somename', 'host': 'localhost', 'port': 0}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.blocking, False)
sock.bind_and_listen()
self.assertTrue(is_nonblock(sock.fileno()))
sock.close()
# test when true
config = {'name': 'somename', 'host': 'localhost', 'port': 0,
'blocking': True}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.blocking, True)
sock.bind_and_listen()
self.assertFalse(is_nonblock(sock.fileno()))
sock.close()
def test_reuseport_supported(self):
config = {'name': '', 'host': 'localhost', 'port': 0,
'so_reuseport': True}
sock = CircusSocket.load_from_config(config)
sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT)
self.assertEqual(sock.so_reuseport, True)
self.assertNotEqual(sockopt, 0)
def test_bind_to_interface(self):
config = {"name": "", "host": "localhost", "port": 0, "interface": "lo"}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.interface, config["interface"])
sock.setsockopt = mock.Mock()
try:
sock.bind_and_listen()
sock.setsockopt.assert_any_call(socket.SOL_SOCKET, IN.SO_BINDTODEVICE, config["interface"] + "\0")
finally:
sock.close()
def test_load_from_config_umask(self):
fd, sockfile = tempfile.mkstemp()
os.close(fd)
os.remove(sockfile)
config = {'name': 'somename', 'path': sockfile, 'umask': 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def test_load_from_config_umask(self):
fd, sockfile = tempfile.mkstemp()
os.close(fd)
os.remove(sockfile)
config = {"name": "somename", "path": sockfile, "umask": 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def test_inet6(self):
config = {"name": "", "host": "::1", "port": 0, "family": "AF_INET6"}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.host, config["host"])
self.assertEqual(sock.port, config["port"])
sock.setsockopt = mock.Mock()
try:
sock.bind_and_listen()
# we should have got a port set
self.assertNotEqual(sock.port, 0)
finally:
sock.close()
def load_from_config(cls, config_file, loop=None):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get("watchers", []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get("sockets", []):
sockets.append(CircusSocket.load_from_config(socket))
httpd = cfg.get("httpd", False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error("You need to install circus-web")
sys.exit(1)
# creating arbiter
arbiter = cls(
watchers,
cfg["endpoint"],
cfg["pubsub_endpoint"],
check_delay=cfg.get("check_delay", 1.0),
prereload_fn=cfg.get("prereload_fn"),
statsd=cfg.get("statsd", False),
stats_endpoint=cfg.get("stats_endpoint"),
multicast_endpoint=cfg.get("multicast_endpoint"),
plugins=cfg.get("plugins"),
sockets=sockets,
warmup_delay=cfg.get("warmup_delay", 0),
httpd=httpd,
loop=loop,
httpd_host=cfg.get("httpd_host", "localhost"),
httpd_port=cfg.get("httpd_port", 8080),
debug=cfg.get("debug", False),
ssh_server=cfg.get("ssh_server", None),
pidfile=cfg.get("pidfile", None),
loglevel=cfg.get("loglevel", None),
logoutput=cfg.get("logoutput", None),
fqdn_prefix=cfg.get("fqdn_prefix", None),
umask=cfg["umask"],
)
# store the cfg which will be used, so it can be used later
# for checking if the cfg has been changed
arbiter._cfg = cls.get_arbiter_config(cfg)
arbiter.config_file = config_file
return arbiter
def test_inet6(self):
config = {'name': '', 'host': '::1', 'port': 0,
'family': 'AF_INET6'}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.host, config['host'])
self.assertEqual(sock.port, config['port'])
sock.setsockopt = mock.Mock()
try:
sock.bind_and_listen()
# we should have got a port set
self.assertNotEqual(sock.port, 0)
finally:
sock.close()
def test_bind_to_interface(self):
config = {'name': '', 'host': 'localhost', 'port': 0,
'interface': 'lo'}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.interface, config['interface'])
sock.setsockopt = mock.Mock()
try:
sock.bind_and_listen()
sock.setsockopt.assert_any_call(socket.SOL_SOCKET,
IN.SO_BINDTODEVICE, config['interface'] + '\0')
finally:
sock.close()
def test_reuseport_unsupported(self):
config = {"name": "", "host": "localhost", "port": 0, "so_reuseport": True}
saved = None
try:
if hasattr(socket, "SO_REUSEPORT"):
saved = socket.SO_REUSEPORT
del socket.SO_REUSEPORT
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.so_reuseport, False)
finally:
if saved is not None:
socket.SO_REUSEPORT = saved
def load_from_config(cls, config_file, loop=None):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket_ in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket_))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
statsd=cfg.get('statsd', False),
stats_endpoint=cfg.get('stats_endpoint'),
papa_endpoint=cfg.get('papa_endpoint'),
multicast_endpoint=cfg.get('multicast_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
loop=loop,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
debug_gc=cfg.get('debug_gc', False),
ssh_server=cfg.get('ssh_server', None),
pidfile=cfg.get('pidfile', None),
loglevel=cfg.get('loglevel', None),
logoutput=cfg.get('logoutput', None),
loggerconfig=cfg.get('loggerconfig', None),
fqdn_prefix=cfg.get('fqdn_prefix', None),
umask=cfg['umask'],
endpoint_owner=cfg.get('endpoint_owner', None))
# store the cfg which will be used, so it can be used later
# for checking if the cfg has been changed
arbiter._cfg = cls.get_arbiter_config(cfg)
arbiter.config_file = config_file
return arbiter
def test_set_inheritable(self):
sockfile = self._get_tmp_filename()
sock = CircusSocket('somename', path=sockfile, umask=0)
try:
sock.bind_and_listen()
self.assertTrue(sock.get_inheritable())
finally:
sock.close()
def test_reuseport_supported(self):
config = {"name": "", "host": "localhost", "port": 0, "so_reuseport": True}
sock = CircusSocket.load_from_config(config)
try:
sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT)
except socket.error:
# see #699
return
finally:
sock.close()
self.assertEqual(sock.so_reuseport, True)
self.assertNotEqual(sockopt, 0)
def test_reuseport_unsupported(self):
config = {'name': '', 'host': 'localhost', 'port': 0,
'so_reuseport': True}
saved = None
try:
if hasattr(socket, 'SO_REUSEPORT'):
saved = socket.SO_REUSEPORT
del socket.SO_REUSEPORT
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.so_reuseport, False)
finally:
if saved is not None:
socket.SO_REUSEPORT = saved
def test_issue310(self):
"""
https://github.com/mozilla-services/circus/pull/310
Allow $(circus.sockets.name) to be used in args.
"""
conf = get_config(_CONF["issue310"])
watcher = Watcher.load_from_config(conf["watchers"][0])
socket = CircusSocket.load_from_config(conf["sockets"][0])
try:
watcher.initialize(None, {"web": socket}, None)
if IS_WINDOWS:
# We can't close the sockets on Windows as we
# are redirecting stdout
watcher.use_sockets = True
process = Process(
watcher._nextwid,
watcher.cmd,
args=watcher.args,
working_dir=watcher.working_dir,
shell=watcher.shell,
uid=watcher.uid,
gid=watcher.gid,
env=watcher.env,
rlimits=watcher.rlimits,
spawn=False,
executable=watcher.executable,
use_fds=watcher.use_sockets,
watcher=watcher,
)
sockets_fds = watcher._get_sockets_fds()
formatted_args = process.format_args(sockets_fds=sockets_fds)
fd = sockets_fds["web"]
self.assertEqual(formatted_args, ["foo", "--fd", str(fd)])
finally:
socket.close()
def test_issue310(self):
'''
https://github.com/mozilla-services/circus/pull/310
Allow $(circus.sockets.name) to be used in args.
'''
conf = get_config(_CONF['issue310'])
watcher = Watcher.load_from_config(conf['watchers'][0])
socket = CircusSocket.load_from_config(conf['sockets'][0])
watcher.initialize(None, {'web': socket}, None)
process = Process(watcher._nextwid, watcher.cmd,
args=watcher.args, working_dir=watcher.working_dir,
shell=watcher.shell, uid=watcher.uid,
gid=watcher.gid, env=watcher.env,
rlimits=watcher.rlimits, spawn=False,
executable=watcher.executable,
use_fds=watcher.use_sockets, watcher=watcher)
fd = watcher._get_sockets_fds()['web']
formatted_args = process.format_args()
self.assertEquals(formatted_args,
['foo', '--fd', str(fd)])
def serve_development(
bento_identifier: str,
working_dir: str,
port: int = Provide[DeploymentContainer.api_server_config.port],
host: str = Provide[DeploymentContainer.api_server_config.host],
backlog: int = Provide[DeploymentContainer.api_server_config.backlog],
with_ngrok: bool = False,
reload: bool = False,
reload_delay: float = 0.25,
) -> None:
working_dir = os.path.realpath(os.path.expanduser(working_dir))
from circus.sockets import CircusSocket # type: ignore
from circus.watcher import Watcher # type: ignore
watchers: t.List[Watcher] = []
if with_ngrok:
watchers.append(
Watcher(
name="ngrok",
cmd=sys.executable,
args=[
"-m",
SCRIPT_NGROK,
],
copy_env=True,
numprocesses=1,
stop_children=True,
working_dir=working_dir,
))
circus_socket_map: t.Dict[str, CircusSocket] = {}
circus_socket_map["_bento_api_server"] = CircusSocket(
name="_bento_api_server",
host=host,
port=port,
backlog=backlog,
)
watchers.append(
Watcher(
name="dev_api_server",
cmd=sys.executable,
args=[
"-m",
SCRIPT_DEV_API_SERVER,
bento_identifier,
"fd://$(circus.sockets._bento_api_server)",
"--working-dir",
working_dir,
] + (["--reload", "--reload-delay", f"{reload_delay}"]
if reload else []),
copy_env=True,
numprocesses=1,
stop_children=True,
use_sockets=True,
working_dir=working_dir,
))
arbiter = create_standalone_arbiter(
watchers,
sockets=list(circus_socket_map.values()),
)
_ensure_prometheus_dir()
arbiter.start(
cb=lambda _: logger.info( # type: ignore
f'Starting development BentoServer from "{bento_identifier}" '
f"running on http://{host}:{port} (Press CTRL+C to quit)"), )
def test_load_from_config_unknown_proto(self):
"""Unknown proto in the config raises an error."""
config = {'name': '', 'proto': 'foo'}
with self.assertRaises(socket.error):
sock = CircusSocket.load_from_config(config)
def main():
#configure_logger(logger, 'DEBUG')
config_mod = 'circus_settings'
if len(sys.argv) > 1:
config_mod = sys.argv[1]
config = import_module(config_mod)
for p in config.PATHS:
ap = os.path.abspath(p)
if ap not in os.sys.path:
os.sys.path.append(ap)
watchers = []
sockets = []
for s in config.SERVERS:
w = Watcher(gc(s, SERVER_NAME),
gc(s, SERVER_PROGRAM),
gc(s, SERVER_ARGS),
numprocesses=gc(s, SERVER_WORKERS, 1),
working_dir=gc(s, SERVER_WORKING_DIR, './'),
env=gc(s, SERVER_ENV, dict()),
copy_env=True,
copy_path=True,
use_sockets=True)
watchers.append(w)
sock_port = gc(s, SERVER_PORT)
if sock_port is not None:
sock_name = gc(s, SERVER_NAME)
sock_host = gc(s, SERVER_HOST, '127.0.0.1')
sock = CircusSocket(sock_name, host=sock_host, port=sock_port)
sockets.append(sock)
for sock in sockets:
print '>> %s'%(sock,)
try:
WANT_WEB = getattr(config, 'WANT_WEB')
except Exception:
WANT_WEB = True
if HAS_WEB and WANT_WEB:
arbiter = Arbiter(watchers, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
sockets=sockets,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
multicast_endpoint=DEFAULT_ENDPOINT_MULTICAST,
statsd=True,
httpd=True,
httpd_port=9999)
else:
arbiter = Arbiter(watchers, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB,
sockets=sockets,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
multicast_endpoint=DEFAULT_ENDPOINT_MULTICAST)
arbiter.start()
def test_load_from_config_no_proto(self):
"""When no proto in the config, the default (0) is used."""
config = {'name': ''}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.proto, 0)
sock.close()
def test_socket(self):
sock = CircusSocket('somename', 'localhost', 0)
try:
sock.bind_and_listen()
finally:
sock.close()
def load_from_config(cls, config_file, loop=None):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
try:
_watcher = Watcher.load_from_config(watcher)
except Exception as e:
logger.warning('Failed to create watcher', exc_info=e)
else:
watchers.append(_watcher)
sockets = []
for socket_ in cfg.get('sockets', []):
try:
_socket = CircusSocket.load_from_config(socket_)
except Exception as e:
logger.warning('Failed to create socket', exc_info=e)
else:
sockets.append(_socket)
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
statsd=cfg.get('statsd', False),
stats_endpoint=cfg.get('stats_endpoint'),
papa_endpoint=cfg.get('papa_endpoint'),
multicast_endpoint=cfg.get('multicast_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
loop=loop,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
debug_gc=cfg.get('debug_gc', False),
ssh_server=cfg.get('ssh_server', None),
pidfile=cfg.get('pidfile', None),
loglevel=cfg.get('loglevel', None),
logoutput=cfg.get('logoutput', None),
loggerconfig=cfg.get('loggerconfig', None),
fqdn_prefix=cfg.get('fqdn_prefix', None),
umask=cfg['umask'],
endpoint_owner=cfg.get('endpoint_owner', None))
# store the cfg which will be used, so it can be used later
# for checking if the cfg has been changed
arbiter._cfg = cls.get_arbiter_config(cfg)
arbiter.config_file = config_file
return arbiter
def __init__(self,
watchers,
endpoint,
pubsub_endpoint,
check_delay=1.0,
prereload_fn=None,
context=None,
loop=None,
statsd=False,
stats_endpoint=None,
statsd_close_outputs=False,
multicast_endpoint=None,
plugins=None,
sockets=None,
warmup_delay=0,
httpd=False,
httpd_host='localhost',
httpd_port=8080,
httpd_close_outputs=False,
debug=False,
debug_gc=False,
ssh_server=None,
proc_name='circusd',
pidfile=None,
loglevel=None,
logoutput=None,
fqdn_prefix=None,
umask=None,
endpoint_owner=None):
self.watchers = watchers
self.endpoint = endpoint
self.check_delay = check_delay
self.prereload_fn = prereload_fn
self.pubsub_endpoint = pubsub_endpoint
self.multicast_endpoint = multicast_endpoint
self.proc_name = proc_name
self.ssh_server = ssh_server
self.evpub_socket = None
self.pidfile = pidfile
self.loglevel = loglevel
self.logoutput = logoutput
self.umask = umask
self.endpoint_owner = endpoint_owner
try:
# getfqdn appears to fail in Python3.3 in the unittest
# framework so fall back to gethostname
socket_fqdn = socket.getfqdn()
except KeyError:
socket_fqdn = socket.gethostname()
if fqdn_prefix is None:
fqdn = socket_fqdn
else:
fqdn = '{}@{}'.format(fqdn_prefix, socket_fqdn)
self.fqdn = fqdn
self.ctrl = self.loop = None
self._provided_loop = False
self.socket_event = False
if loop is not None:
self._provided_loop = True
self.loop = loop
# initialize zmq context
self._init_context(context)
self.pid = os.getpid()
self._watchers_names = {}
self._stopping = False
self._restarting = False
self.debug = debug
self._exclusive_running_command = None
if self.debug:
self.stdout_stream = self.stderr_stream = {'class': 'StdoutStream'}
else:
self.stdout_stream = self.stderr_stream = None
self.debug_gc = debug_gc
if debug_gc:
gc.set_debug(gc.DEBUG_LEAK)
# initializing circusd-stats as a watcher when configured
self.statsd = statsd
self.stats_endpoint = stats_endpoint
if self.statsd:
cmd = "%s -c 'from circus import stats; stats.main()'" % \
sys.executable
cmd += ' --endpoint %s' % self.endpoint
cmd += ' --pubsub %s' % self.pubsub_endpoint
cmd += ' --statspoint %s' % self.stats_endpoint
if ssh_server is not None:
cmd += ' --ssh %s' % ssh_server
if debug:
cmd += ' --log-level DEBUG'
stats_watcher = Watcher('circusd-stats',
cmd,
use_sockets=True,
singleton=True,
stdout_stream=self.stdout_stream,
stderr_stream=self.stderr_stream,
copy_env=True,
copy_path=True,
close_child_stderr=statsd_close_outputs,
close_child_stdout=statsd_close_outputs)
self.watchers.append(stats_watcher)
# adding the httpd
if httpd:
# adding the socket
httpd_socket = CircusSocket(name='circushttpd',
host=httpd_host,
port=httpd_port)
if sockets is None:
sockets = [httpd_socket]
else:
sockets.append(httpd_socket)
cmd = ("%s -c 'from circusweb import circushttpd; "
"circushttpd.main()'") % sys.executable
cmd += ' --endpoint %s' % self.endpoint
cmd += ' --fd $(circus.sockets.circushttpd)'
if ssh_server is not None:
cmd += ' --ssh %s' % ssh_server
# Adding the watcher
httpd_watcher = Watcher('circushttpd',
cmd,
use_sockets=True,
singleton=True,
stdout_stream=self.stdout_stream,
stderr_stream=self.stderr_stream,
copy_env=True,
copy_path=True,
close_child_stderr=httpd_close_outputs,
close_child_stdout=httpd_close_outputs)
self.watchers.append(httpd_watcher)
# adding each plugin as a watcher
ch_stderr = self.stderr_stream is None
ch_stdout = self.stdout_stream is None
if plugins is not None:
for plugin in plugins:
fqn = plugin['use']
cmd = get_plugin_cmd(plugin,
self.endpoint,
self.pubsub_endpoint,
self.check_delay,
ssh_server,
debug=self.debug)
plugin_cfg = dict(cmd=cmd,
priority=1,
singleton=True,
stdout_stream=self.stdout_stream,
stderr_stream=self.stderr_stream,
copy_env=True,
copy_path=True,
close_child_stderr=ch_stderr,
close_child_stdout=ch_stdout)
plugin_cfg.update(plugin)
if 'name' not in plugin_cfg:
plugin_cfg['name'] = fqn
plugin_watcher = Watcher.load_from_config(plugin_cfg)
self.watchers.append(plugin_watcher)
self.sockets = CircusSockets(sockets)
self.warmup_delay = warmup_delay
def reload_from_config(self, config_file=None, inside_circusd=False):
new_cfg = get_config(config_file if config_file else self.config_file)
# if arbiter is changed, reload everything
if self.get_arbiter_config(new_cfg) != self._cfg:
yield self._restart(inside_circusd=inside_circusd)
return
ignore_sn = set(['circushttpd'])
ignore_wn = set(['circushttpd', 'circusd-stats'])
# Gather socket names.
current_sn = set([i.name for i in self.sockets.values()]) - ignore_sn
new_sn = set([i['name'] for i in new_cfg.get('sockets', [])])
added_sn = new_sn - current_sn
deleted_sn = current_sn - new_sn
maybechanged_sn = current_sn - deleted_sn
changed_sn = set([])
wn_with_changed_socket = set([])
wn_with_deleted_socket = set([])
# get changed sockets
for n in maybechanged_sn:
s = self.get_socket(n)
if self.get_socket_config(new_cfg, n) != s._cfg:
changed_sn.add(n)
# just delete the socket and add it again
deleted_sn.add(n)
added_sn.add(n)
# Get the watchers whichs use these, so they could be
# deleted and added also
for w in self.iter_watchers():
if 'circus.sockets.%s' % n.lower() in w.cmd:
wn_with_changed_socket.add(w.name)
# get deleted sockets
for n in deleted_sn:
s = self.get_socket(n)
s.close()
# Get the watchers whichs use these, these should not be
# active anymore
for w in self.iter_watchers():
if 'circus.sockets.%s' % n.lower() in w.cmd:
wn_with_deleted_socket.add(w.name)
del self.sockets[s.name]
# get added sockets
for n in added_sn:
socket_config = self.get_socket_config(new_cfg, n)
s = CircusSocket.load_from_config(socket_config)
s.bind_and_listen()
self.sockets[s.name] = s
if added_sn or deleted_sn:
# make sure all existing watchers get the new sockets in
# their attributes and get the old removed
# XXX: is this necessary? self.sockets is an mutable
# object
for watcher in self.iter_watchers():
# XXX: What happens as initalize is called on a
# running watcher?
watcher.initialize(self.evpub_socket, self.sockets, self)
# Gather watcher names.
current_wn = set([i.name for i in self.iter_watchers()]) - ignore_wn
new_wn = set([i['name'] for i in new_cfg.get('watchers', [])])
new_wn = new_wn | set([i['name'] for i in new_cfg.get('plugins', [])])
added_wn = (new_wn - current_wn) | wn_with_changed_socket
deleted_wn = current_wn - new_wn - wn_with_changed_socket
maybechanged_wn = current_wn - deleted_wn
changed_wn = set([])
if wn_with_deleted_socket and wn_with_deleted_socket not in new_wn:
raise ValueError('Watchers %s uses a socket which is deleted' %
wn_with_deleted_socket)
# get changed watchers
for n in maybechanged_wn:
w = self.get_watcher(n)
new_watcher_cfg = (self.get_watcher_config(new_cfg, n)
or self.get_plugin_config(new_cfg, n))
old_watcher_cfg = w._cfg.copy()
if 'env' in new_watcher_cfg:
new_watcher_cfg['env'] = parse_env_dict(new_watcher_cfg['env'])
# discarding env exceptions
for key in _ENV_EXCEPTIONS:
if 'env' in new_watcher_cfg and key in new_watcher_cfg['env']:
del new_watcher_cfg['env'][key]
if 'env' in new_watcher_cfg and key in old_watcher_cfg['env']:
del old_watcher_cfg['env'][key]
diff = DictDiffer(new_watcher_cfg, old_watcher_cfg).changed()
if diff == set(['numprocesses']):
# if nothing but the number of processes is
# changed, just changes this
w.set_numprocesses(int(new_watcher_cfg['numprocesses']))
changed = False
else:
changed = len(diff) > 0
if changed:
# Others things are changed. Just delete and add the watcher.
changed_wn.add(n)
deleted_wn.add(n)
added_wn.add(n)
# delete watchers
for n in deleted_wn:
w = self.get_watcher(n)
yield w._stop()
del self._watchers_names[w.name.lower()]
self.watchers.remove(w)
# add watchers
for n in added_wn:
new_watcher_cfg = (self.get_plugin_config(new_cfg, n)
or self.get_watcher_config(new_cfg, n))
w = Watcher.load_from_config(new_watcher_cfg)
w.initialize(self.evpub_socket, self.sockets, self)
yield self.start_watcher(w)
self.watchers.append(w)
self._watchers_names[w.name.lower()] = w
def serve_production(
bento_identifier: str,
working_dir: str,
port: int = Provide[DeploymentContainer.api_server_config.port],
host: str = Provide[DeploymentContainer.api_server_config.host],
backlog: int = Provide[DeploymentContainer.api_server_config.backlog],
app_workers: t.Optional[int] = None,
) -> None:
working_dir = os.path.realpath(os.path.expanduser(working_dir))
svc = load(bento_identifier,
working_dir=working_dir,
change_global_cwd=True)
from circus.sockets import CircusSocket # type: ignore
from circus.watcher import Watcher # type: ignore
watchers: t.List[Watcher] = []
circus_socket_map: t.Dict[str, CircusSocket] = {}
runner_bind_map: t.Dict[str, str] = {}
uds_path = None
if psutil.POSIX:
# use AF_UNIX sockets for Circus
uds_path = tempfile.mkdtemp()
for runner_name, runner in svc.runners.items():
sockets_path = os.path.join(uds_path, f"{id(runner)}.sock")
assert len(sockets_path) < MAX_AF_UNIX_PATH_LENGTH
runner_bind_map[runner_name] = path_to_uri(sockets_path)
circus_socket_map[runner_name] = CircusSocket(
name=runner_name,
path=sockets_path,
backlog=backlog,
)
watchers.append(
Watcher(
name=f"runner_{runner_name}",
cmd=sys.executable,
args=[
"-m",
SCRIPT_RUNNER,
bento_identifier,
runner_name,
f"fd://$(circus.sockets.{runner_name})",
"--working-dir",
working_dir,
],
copy_env=True,
stop_children=True,
working_dir=working_dir,
use_sockets=True,
numprocesses=runner.num_replica,
))
elif psutil.WINDOWS:
# Windows doesn't (fully) support AF_UNIX sockets
with contextlib.ExitStack() as port_stack:
for runner_name, runner in svc.runners.items():
runner_port = port_stack.enter_context(reserve_free_port())
runner_host = "127.0.0.1"
runner_bind_map[
runner_name] = f"tcp://{runner_host}:{runner_port}"
circus_socket_map[runner_name] = CircusSocket(
name=runner_name,
host=runner_host,
port=runner_port,
backlog=backlog,
)
watchers.append(
Watcher(
name=f"runner_{runner_name}",
cmd=sys.executable,
args=[
"-m",
SCRIPT_RUNNER,
bento_identifier,
runner_name,
f"fd://$(circus.sockets.{runner_name})",
"--working-dir",
working_dir,
],
copy_env=True,
stop_children=True,
use_sockets=True,
working_dir=working_dir,
numprocesses=runner.num_replica,
))
port_stack.enter_context(
reserve_free_port()) # reserve one more to avoid conflicts
else:
raise NotImplementedError("Unsupported platform: {}".format(
sys.platform))
logger.debug("Runner map: %s", runner_bind_map)
circus_socket_map["_bento_api_server"] = CircusSocket(
name="_bento_api_server",
host=host,
port=port,
backlog=backlog,
)
watchers.append(
Watcher(
name="api_server",
cmd=sys.executable,
args=[
"-m",
SCRIPT_API_SERVER,
bento_identifier,
"fd://$(circus.sockets._bento_api_server)",
"--runner-map",
json.dumps(runner_bind_map),
"--working-dir",
working_dir,
"--backlog",
f"{backlog}",
],
copy_env=True,
numprocesses=app_workers or 1,
stop_children=True,
use_sockets=True,
working_dir=working_dir,
))
arbiter = create_standalone_arbiter(
watchers=watchers,
sockets=list(circus_socket_map.values()),
)
_ensure_prometheus_dir()
try:
arbiter.start(
cb=lambda _: logger.info( # type: ignore
f'Starting production BentoServer from "bento_identifier" '
f"running on http://{host}:{port} (Press CTRL+C to quit)"), )
finally:
if uds_path is not None:
shutil.rmtree(uds_path)
def __init__(self, watchers, endpoint, pubsub_endpoint, check_delay=.5,
prereload_fn=None, context=None, loop=None,
stats_endpoint=None, plugins=None, sockets=None,
warmup_delay=0, httpd=False, httpd_host='localhost',
httpd_port=8080, debug=False, ssh_server=None,
proc_name='circusd'):
self.watchers = watchers
self.endpoint = endpoint
self.check_delay = check_delay
self.prereload_fn = prereload_fn
self.pubsub_endpoint = pubsub_endpoint
self.proc_name = proc_name
self.ctrl = self.loop = None
self.socket_event = False
# initialize zmq context
self.context = context or zmq.Context.instance()
self.pid = os.getpid()
self._watchers_names = {}
self.alive = True
self._lock = RLock()
self.debug = debug
if self.debug:
stdout_stream = stderr_stream = {'class': 'StdoutStream'}
else:
stdout_stream = stderr_stream = None
# initializing circusd-stats as a watcher when configured
self.stats_endpoint = stats_endpoint
if self.stats_endpoint is not None:
cmd = "%s -c 'from circus import stats; stats.main()'" % \
sys.executable
cmd += ' --endpoint %s' % self.endpoint
cmd += ' --pubsub %s' % self.pubsub_endpoint
cmd += ' --statspoint %s' % self.stats_endpoint
if ssh_server is not None:
cmd += ' --ssh %s' % ssh_server
if debug:
cmd += ' --log-level DEBUG'
stats_watcher = Watcher('circusd-stats', cmd, use_sockets=True,
singleton=True,
stdout_stream=stdout_stream,
stderr_stream=stderr_stream,
copy_env=True, copy_path=True)
self.watchers.append(stats_watcher)
# adding the httpd
if httpd:
cmd = ("%s -c 'from circusweb import circushttpd; "
"circushttpd.main()'") % sys.executable
cmd += ' --endpoint %s' % self.endpoint
cmd += ' --fd $(circus.sockets.circushttpd)'
if ssh_server is not None:
cmd += ' --ssh %s' % ssh_server
httpd_watcher = Watcher('circushttpd', cmd, use_sockets=True,
singleton=True,
stdout_stream=stdout_stream,
stderr_stream=stderr_stream,
copy_env=True, copy_path=True)
self.watchers.append(httpd_watcher)
httpd_socket = CircusSocket(name='circushttpd', host=httpd_host,
port=httpd_port)
# adding the socket
if sockets is None:
sockets = [httpd_socket]
else:
sockets.append(httpd_socket)
# adding each plugin as a watcher
if plugins is not None:
for plugin in plugins:
fqnd = plugin['use']
name = 'plugin:%s' % fqnd.replace('.', '-')
cmd = get_plugin_cmd(plugin, self.endpoint,
self.pubsub_endpoint, self.check_delay,
ssh_server, debug=self.debug)
plugin_watcher = Watcher(name, cmd, priority=1, singleton=True,
stdout_stream=stdout_stream,
stderr_stream=stderr_stream,
copy_env=True, copy_path=True)
self.watchers.append(plugin_watcher)
self.sockets = CircusSockets(sockets)
self.warmup_delay = warmup_delay
self.loop = ioloop.IOLoop.instance()
self.ctrl = Controller(self.endpoint, self.context, self.loop, self,
self.check_delay)
def __init__(self,
watchers,
endpoint,
pubsub_endpoint,
check_delay=.5,
prereload_fn=None,
context=None,
loop=None,
statsd=False,
stats_endpoint=None,
statsd_close_outputs=False,
multicast_endpoint=None,
plugins=None,
sockets=None,
warmup_delay=0,
httpd=False,
httpd_host='localhost',
httpd_port=8080,
httpd_close_outputs=False,
debug=False,
ssh_server=None,
proc_name='circusd',
pidfile=None,
loglevel=None,
logoutput=None,
fqdn_prefix=None):
self.watchers = watchers
self.endpoint = endpoint
self.check_delay = check_delay
self.prereload_fn = prereload_fn
self.pubsub_endpoint = pubsub_endpoint
self.multicast_endpoint = multicast_endpoint
self.proc_name = proc_name
self.ssh_server = ssh_server
self.pidfile = pidfile
self.loglevel = loglevel
self.logoutput = logoutput
socket_fqdn = socket.getfqdn()
if fqdn_prefix is None:
fqdn = socket_fqdn
else:
fqdn = '{}@{}'.format(fqdn_prefix, socket_fqdn)
self.fqdn = fqdn
self.ctrl = self.loop = None
self.socket_event = False
# initialize zmq context
self._init_context(context)
self.pid = os.getpid()
self._watchers_names = {}
self.alive = True
self._lock = RLock()
self.debug = debug
if self.debug:
self.stdout_stream = self.stderr_stream = {'class': 'StdoutStream'}
else:
self.stdout_stream = self.stderr_stream = None
# initializing circusd-stats as a watcher when configured
self.statsd = statsd
self.stats_endpoint = stats_endpoint
if self.statsd:
cmd = "%s -c 'from circus import stats; stats.main()'" % \
sys.executable
cmd += ' --endpoint %s' % self.endpoint
cmd += ' --pubsub %s' % self.pubsub_endpoint
cmd += ' --statspoint %s' % self.stats_endpoint
if ssh_server is not None:
cmd += ' --ssh %s' % ssh_server
if debug:
cmd += ' --log-level DEBUG'
stats_watcher = Watcher('circusd-stats',
cmd,
use_sockets=True,
singleton=True,
stdout_stream=self.stdout_stream,
stderr_stream=self.stderr_stream,
copy_env=True,
copy_path=True,
close_child_stderr=statsd_close_outputs,
close_child_stdout=statsd_close_outputs)
self.watchers.append(stats_watcher)
# adding the httpd
if httpd:
cmd = ("%s -c 'from circusweb import circushttpd; "
"circushttpd.main()'") % sys.executable
cmd += ' --endpoint %s' % self.endpoint
cmd += ' --fd $(circus.sockets.circushttpd)'
if ssh_server is not None:
cmd += ' --ssh %s' % ssh_server
httpd_watcher = Watcher('circushttpd',
cmd,
use_sockets=True,
singleton=True,
stdout_stream=self.stdout_stream,
stderr_stream=self.stderr_stream,
copy_env=True,
copy_path=True,
close_child_stderr=httpd_close_outputs,
close_child_stdout=httpd_close_outputs)
self.watchers.append(httpd_watcher)
httpd_socket = CircusSocket(name='circushttpd',
host=httpd_host,
port=httpd_port)
# adding the socket
if sockets is None:
sockets = [httpd_socket]
else:
sockets.append(httpd_socket)
# adding each plugin as a watcher
ch_stderr = self.stderr_stream is None
ch_stdout = self.stdout_stream is None
if plugins is not None:
for plugin in plugins:
fqn = plugin['use']
cmd = get_plugin_cmd(plugin,
self.endpoint,
self.pubsub_endpoint,
self.check_delay,
ssh_server,
debug=self.debug)
plugin_cfg = dict(cmd=cmd,
priority=1,
singleton=True,
stdout_stream=self.stdout_stream,
stderr_stream=self.stderr_stream,
copy_env=True,
copy_path=True,
close_child_stderr=ch_stderr,
close_child_stdout=ch_stdout)
plugin_cfg.update(plugin)
if 'name' not in plugin_cfg:
plugin_cfg['name'] = fqn
plugin_watcher = Watcher.load_from_config(plugin_cfg)
self.watchers.append(plugin_watcher)
self.sockets = CircusSockets(sockets)
self.warmup_delay = warmup_delay
