def load_from_config(cls, config_file):
cfg = get_config(config_file)
# hack reload ioloop to use the monkey patched version
reload(ioloop)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=cfg.get('httpd', False),
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080))
return arbiter
def load_from_config(cls, config_file):
cfg = get_config(config_file)
# hack reload ioloop to use the monkey patched version
reload(ioloop)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
# creating arbiter
arbiter = cls(watchers,
cfg['endpoint'],
cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'),
sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=cfg.get('httpd', False),
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
stream_backend=cfg.get('stream_backend', 'thread'),
ssh_server=cfg.get('ssh_server', None))
return arbiter
def load_from_config(cls, config_file):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
ssh_server=cfg.get('ssh_server', None))
return arbiter
def load_from_config(cls, config_file):
cfg = get_config(config_file)
# hack reload ioloop to use the monkey patched version
reload(ioloop)
watchers = []
for watcher in cfg.get("watchers", []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get("sockets", []):
sockets.append(CircusSocket.load_from_config(socket))
# creating arbiter
arbiter = cls(
watchers,
cfg["endpoint"],
cfg["pubsub_endpoint"],
check_delay=cfg.get("check_delay", 1.0),
prereload_fn=cfg.get("prereload_fn"),
stats_endpoint=cfg.get("stats_endpoint"),
plugins=cfg.get("plugins"),
sockets=sockets,
)
return arbiter
def _get_sockets_fds(self):
"""Returns sockets dict. If this worker's cmd indicates use of
a SO_REUSEPORT socket, a new socket is created and bound. This
new socket's FD replaces original socket's FD in returned dict.
This method populates `self._sockets` list. This list should be
let go after `fork()`.
"""
sockets_fds = None
if self.watcher is not None and self.watcher.sockets is not None:
sockets_fds = self.watcher._get_sockets_fds()
reuseport_sockets = tuple((sn, s) for (sn, s)
in self.watcher.sockets.items()
if s.so_reuseport)
for sn, s in reuseport_sockets:
# watcher.cmd uses this reuseport socket
if 'circus.sockets.%s' % sn in self.watcher.cmd:
sock = CircusSocket.load_from_config(s._cfg)
sock.bind_and_listen()
# replace original socket's fd
sockets_fds[sn] = sock.fileno()
# keep new socket until fork returns
self._sockets.append(sock)
return sockets_fds
def _get_sockets_fds(self):
"""Returns sockets dict. If this worker's cmd indicates use of
a SO_REUSEPORT socket, a new socket is created and bound. This
new socket's FD replaces original socket's FD in returned dict.
This method populates `self._sockets` list. This list should be
let go after `fork()`.
"""
sockets_fds = None
if self.watcher is not None and self.watcher.sockets is not None:
sockets_fds = self.watcher._get_sockets_fds()
reuseport_sockets = tuple(
(sn, s) for (sn, s) in self.watcher.sockets.items()
if s.so_reuseport)
for sn, s in reuseport_sockets:
# watcher.cmd uses this reuseport socket
if 'circus.sockets.%s' % sn in self.watcher.cmd:
sock = CircusSocket.load_from_config(s._cfg)
sock.bind_and_listen()
# replace original socket's fd
sockets_fds[sn] = sock.fileno()
# keep new socket until fork returns
self._sockets.append(sock)
return sockets_fds
def test_issue310(self):
"""
https://github.com/mozilla-services/circus/pull/310
Allow $(circus.sockets.name) to be used in args.
"""
conf = get_config(_CONF["issue310"])
watcher = Watcher.load_from_config(conf["watchers"][0])
socket = CircusSocket.load_from_config(conf["sockets"][0])
watcher.initialize(None, {"web": socket}, None)
process = Process(
watcher._process_counter,
watcher.cmd,
args=watcher.args,
working_dir=watcher.working_dir,
shell=watcher.shell,
uid=watcher.uid,
gid=watcher.gid,
env=watcher.env,
rlimits=watcher.rlimits,
spawn=False,
executable=watcher.executable,
use_fds=watcher.use_sockets,
watcher=watcher,
)
fd = watcher._get_sockets_fds()["web"]
formatted_args = process.format_args()
self.assertEquals(formatted_args, ["foo", "--fd", str(fd)])
def load_from_config(cls, config_file):
cfg = get_config(config_file)
# hack reload ioloop to use the monkey patched version
reload(ioloop)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
# creating arbiter
arbiter = cls(watchers,
cfg['endpoint'],
cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'),
sockets=sockets)
return arbiter
def test_issue310(self):
'''
https://github.com/mozilla-services/circus/pull/310
Allow $(circus.sockets.name) to be used in args.
'''
conf = get_config(_CONF['issue310'])
watcher = Watcher.load_from_config(conf['watchers'][0])
socket = CircusSocket.load_from_config(conf['sockets'][0])
try:
watcher.initialize(None, {'web': socket}, None)
process = Process(watcher._nextwid, watcher.cmd,
args=watcher.args,
working_dir=watcher.working_dir,
shell=watcher.shell, uid=watcher.uid,
gid=watcher.gid, env=watcher.env,
rlimits=watcher.rlimits, spawn=False,
executable=watcher.executable,
use_fds=watcher.use_sockets,
watcher=watcher)
sockets_fds = watcher._get_sockets_fds()
formatted_args = process.format_args(sockets_fds=sockets_fds)
fd = sockets_fds['web']
self.assertEqual(formatted_args,
['foo', '--fd', str(fd)])
finally:
socket.close()
def load_from_config(cls, config_file):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=cfg.get('httpd', False),
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
stream_backend=cfg.get('stream_backend', 'thread'),
ssh_server=cfg.get('ssh_server', None))
return arbiter
def load_from_config(cls, config_file):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
ssh_server=cfg.get('ssh_server', None))
return arbiter
def test_load_from_config_replace(self):
sockfile = self._get_file()
config = {'name': 'somename', 'path': sockfile, 'replace': False}
sock = CircusSocket.load_from_config(config)
try:
self.assertRaises(OSError, sock.bind_and_listen)
finally:
sock.close()
config = {'name': 'somename', 'path': sockfile, 'replace': True}
sock = CircusSocket.load_from_config(config)
sock.bind_and_listen()
try:
self.assertEqual(sock.replace, True)
finally:
sock.close()
def test_load_from_config_umask(self):
sockfile = self._get_tmp_filename()
config = {"name": "somename", "path": sockfile, "umask": 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def test_load_from_config_umask(self):
sockfile = self._get_tmp_filename()
config = {'name': 'somename', 'path': sockfile, 'umask': 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def test_reuseport_supported(self):
config = {"name": "", "host": "localhost", "port": 0, "so_reuseport": True}
sock = CircusSocket.load_from_config(config)
sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT)
self.assertEqual(sock.so_reuseport, True)
self.assertNotEqual(sockopt, 0)
def test_load_from_config_blocking(self):
# test default to false
config = {'name': 'somename', 'host': 'localhost', 'port': 0}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.blocking, False)
sock.bind_and_listen()
self.assertTrue(is_nonblock(sock.fileno()))
sock.close()
# test when true
config = {'name': 'somename', 'host': 'localhost', 'port': 0,
'blocking': True}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.blocking, True)
sock.bind_and_listen()
self.assertFalse(is_nonblock(sock.fileno()))
sock.close()
def test_load_from_config_umask(self):
sockfile = self._get_tmp_filename()
config = {'name': 'somename', 'path': sockfile, 'umask': 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def test_load_from_config_replace(self):
sockfile = self._get_file()
config = {'name': 'somename', 'path': sockfile, 'replace': False}
sock = CircusSocket.load_from_config(config)
try:
self.assertRaises(OSError, sock.bind_and_listen)
finally:
sock.close()
config = {'name': 'somename', 'path': sockfile, 'replace': True}
sock = CircusSocket.load_from_config(config)
sock.bind_and_listen()
try:
self.assertEqual(sock.replace, True)
finally:
sock.close()
def test_reuseport_supported(self):
config = {'name': '', 'host': 'localhost', 'port': 0,
'so_reuseport': True}
sock = CircusSocket.load_from_config(config)
sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT)
self.assertEqual(sock.so_reuseport, True)
self.assertNotEqual(sockopt, 0)
def test_load_from_config_blocking(self):
# test default to false
config = {'name': 'somename', 'host': 'localhost', 'port': 0}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.blocking, False)
sock.bind_and_listen()
fl = fcntl.fcntl(sock.fileno(), fcntl.F_GETFL)
self.assertTrue(is_nonblock(sock.fileno()))
sock.close()
# test when true
config = {'name': 'somename', 'host': 'localhost', 'port': 0,
'blocking': True}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.blocking, True)
sock.bind_and_listen()
fl2 = fcntl.fcntl(sock.fileno(), fcntl.F_GETFL)
self.assertFalse(is_nonblock(sock.fileno()))
sock.close()
def test_bind_to_interface(self):
config = {"name": "", "host": "localhost", "port": 0, "interface": "lo"}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.interface, config["interface"])
sock.setsockopt = mock.Mock()
try:
sock.bind_and_listen()
sock.setsockopt.assert_any_call(socket.SOL_SOCKET, IN.SO_BINDTODEVICE, config["interface"] + "\0")
finally:
sock.close()
def test_load_from_config_umask(self):
fd, sockfile = tempfile.mkstemp()
os.close(fd)
os.remove(sockfile)
config = {"name": "somename", "path": sockfile, "umask": 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def load_from_config(cls, config_file, loop=None):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket_ in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket_))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers,
cfg['endpoint'],
cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
statsd=cfg.get('statsd', False),
stats_endpoint=cfg.get('stats_endpoint'),
papa_endpoint=cfg.get('papa_endpoint'),
multicast_endpoint=cfg.get('multicast_endpoint'),
plugins=cfg.get('plugins'),
sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
loop=loop,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
debug_gc=cfg.get('debug_gc', False),
ssh_server=cfg.get('ssh_server', None),
pidfile=cfg.get('pidfile', None),
loglevel=cfg.get('loglevel', None),
logoutput=cfg.get('logoutput', None),
loggerconfig=cfg.get('loggerconfig', None),
fqdn_prefix=cfg.get('fqdn_prefix', None),
umask=cfg['umask'],
endpoint_owner=cfg.get('endpoint_owner', None))
# store the cfg which will be used, so it can be used later
# for checking if the cfg has been changed
arbiter._cfg = cls.get_arbiter_config(cfg)
arbiter.config_file = config_file
return arbiter
def test_load_from_config_umask(self):
fd, sockfile = tempfile.mkstemp()
os.close(fd)
os.remove(sockfile)
config = {'name': 'somename', 'path': sockfile, 'umask': 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def test_load_from_config_umask(self):
fd, sockfile = tempfile.mkstemp()
os.close(fd)
os.remove(sockfile)
config = {'name': 'somename', 'path': sockfile, 'umask': 0}
sock = CircusSocket.load_from_config(config)
try:
self.assertEqual(sock.umask, 0)
finally:
sock.close()
def test_inet6(self):
config = {'name': '', 'host': '::1', 'port': 0, 'family': 'AF_INET6'}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.host, config['host'])
self.assertEqual(sock.port, config['port'])
sock.setsockopt = mock.Mock()
try:
sock.bind_and_listen()
# we should have got a port set
self.assertNotEqual(sock.port, 0)
finally:
sock.close()
def test_inet6(self):
config = {"name": "", "host": "::1", "port": 0, "family": "AF_INET6"}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.host, config["host"])
self.assertEqual(sock.port, config["port"])
sock.setsockopt = mock.Mock()
try:
sock.bind_and_listen()
# we should have got a port set
self.assertNotEqual(sock.port, 0)
finally:
sock.close()
def load_from_config(cls, config_file, loop=None):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get("watchers", []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get("sockets", []):
sockets.append(CircusSocket.load_from_config(socket))
httpd = cfg.get("httpd", False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error("You need to install circus-web")
sys.exit(1)
# creating arbiter
arbiter = cls(
watchers,
cfg["endpoint"],
cfg["pubsub_endpoint"],
check_delay=cfg.get("check_delay", 1.0),
prereload_fn=cfg.get("prereload_fn"),
statsd=cfg.get("statsd", False),
stats_endpoint=cfg.get("stats_endpoint"),
multicast_endpoint=cfg.get("multicast_endpoint"),
plugins=cfg.get("plugins"),
sockets=sockets,
warmup_delay=cfg.get("warmup_delay", 0),
httpd=httpd,
loop=loop,
httpd_host=cfg.get("httpd_host", "localhost"),
httpd_port=cfg.get("httpd_port", 8080),
debug=cfg.get("debug", False),
ssh_server=cfg.get("ssh_server", None),
pidfile=cfg.get("pidfile", None),
loglevel=cfg.get("loglevel", None),
logoutput=cfg.get("logoutput", None),
fqdn_prefix=cfg.get("fqdn_prefix", None),
umask=cfg["umask"],
)
# store the cfg which will be used, so it can be used later
# for checking if the cfg has been changed
arbiter._cfg = cls.get_arbiter_config(cfg)
arbiter.config_file = config_file
return arbiter
def test_bind_to_interface(self):
config = {'name': '', 'host': 'localhost', 'port': 0,
'interface': 'lo'}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.interface, config['interface'])
sock.setsockopt = mock.Mock()
try:
sock.bind_and_listen()
sock.setsockopt.assert_any_call(socket.SOL_SOCKET,
IN.SO_BINDTODEVICE, config['interface'] + '\0')
finally:
sock.close()
def test_inet6(self):
config = {'name': '', 'host': '::1', 'port': 0,
'family': 'AF_INET6'}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.host, config['host'])
self.assertEqual(sock.port, config['port'])
sock.setsockopt = mock.Mock()
try:
sock.bind_and_listen()
# we should have got a port set
self.assertNotEqual(sock.port, 0)
finally:
sock.close()
def test_reuseport_unsupported(self):
config = {"name": "", "host": "localhost", "port": 0, "so_reuseport": True}
saved = None
try:
if hasattr(socket, "SO_REUSEPORT"):
saved = socket.SO_REUSEPORT
del socket.SO_REUSEPORT
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.so_reuseport, False)
finally:
if saved is not None:
socket.SO_REUSEPORT = saved
def load_from_config(cls, config_file, loop=None):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket_ in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket_))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
statsd=cfg.get('statsd', False),
stats_endpoint=cfg.get('stats_endpoint'),
papa_endpoint=cfg.get('papa_endpoint'),
multicast_endpoint=cfg.get('multicast_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
loop=loop,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
debug_gc=cfg.get('debug_gc', False),
ssh_server=cfg.get('ssh_server', None),
pidfile=cfg.get('pidfile', None),
loglevel=cfg.get('loglevel', None),
logoutput=cfg.get('logoutput', None),
loggerconfig=cfg.get('loggerconfig', None),
fqdn_prefix=cfg.get('fqdn_prefix', None),
umask=cfg['umask'],
endpoint_owner=cfg.get('endpoint_owner', None))
# store the cfg which will be used, so it can be used later
# for checking if the cfg has been changed
arbiter._cfg = cls.get_arbiter_config(cfg)
arbiter.config_file = config_file
return arbiter
def test_reuseport_supported(self):
config = {"name": "", "host": "localhost", "port": 0, "so_reuseport": True}
sock = CircusSocket.load_from_config(config)
try:
sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT)
except socket.error:
# see #699
return
finally:
sock.close()
self.assertEqual(sock.so_reuseport, True)
self.assertNotEqual(sockopt, 0)
def test_reuseport_unsupported(self):
config = {'name': '', 'host': 'localhost', 'port': 0,
'so_reuseport': True}
saved = None
try:
if hasattr(socket, 'SO_REUSEPORT'):
saved = socket.SO_REUSEPORT
del socket.SO_REUSEPORT
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.so_reuseport, False)
finally:
if saved is not None:
socket.SO_REUSEPORT = saved
def test_reuseport_unsupported(self):
config = {'name': '', 'host': 'localhost', 'port': 0,
'so_reuseport': True}
saved = None
try:
if hasattr(socket, 'SO_REUSEPORT'):
saved = socket.SO_REUSEPORT
del socket.SO_REUSEPORT
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.so_reuseport, False)
finally:
if saved is not None:
socket.SO_REUSEPORT = saved
def test_reuseport_supported(self):
config = {'name': '', 'host': 'localhost', 'port': 0,
'so_reuseport': True}
sock = CircusSocket.load_from_config(config)
try:
sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT)
except socket.error:
# see #699
return
finally:
sock.close()
self.assertEqual(sock.so_reuseport, True)
self.assertNotEqual(sockopt, 0)
def test_issue310(self):
"""
https://github.com/mozilla-services/circus/pull/310
Allow $(circus.sockets.name) to be used in args.
"""
conf = get_config(_CONF["issue310"])
watcher = Watcher.load_from_config(conf["watchers"][0])
socket = CircusSocket.load_from_config(conf["sockets"][0])
try:
watcher.initialize(None, {"web": socket}, None)
if IS_WINDOWS:
# We can't close the sockets on Windows as we
# are redirecting stdout
watcher.use_sockets = True
process = Process(
watcher._nextwid,
watcher.cmd,
args=watcher.args,
working_dir=watcher.working_dir,
shell=watcher.shell,
uid=watcher.uid,
gid=watcher.gid,
env=watcher.env,
rlimits=watcher.rlimits,
spawn=False,
executable=watcher.executable,
use_fds=watcher.use_sockets,
watcher=watcher,
)
sockets_fds = watcher._get_sockets_fds()
formatted_args = process.format_args(sockets_fds=sockets_fds)
fd = sockets_fds["web"]
self.assertEqual(formatted_args, ["foo", "--fd", str(fd)])
finally:
socket.close()
def test_load_from_config_no_proto(self):
"""When no proto in the config, the default (0) is used."""
config = {'name': ''}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.proto, 0)
def reload_from_config(self, config_file=None):
new_cfg = get_config(config_file if config_file else self.config_file)
# if arbiter is changed, reload everything
if self.get_arbiter_config(new_cfg) != self._cfg:
raise ReloadArbiterException
# Gather socket names.
current_sn = set([i.name for i in self.sockets.values()])
new_sn = set([i['name'] for i in new_cfg.get('sockets', [])])
added_sn = new_sn - current_sn
deleted_sn = current_sn - new_sn
maybechanged_sn = current_sn - deleted_sn
changed_sn = set([])
wn_with_changed_socket = set([])
wn_with_deleted_socket = set([])
# get changed sockets
for n in maybechanged_sn:
s = self.get_socket(n)
if self.get_socket_config(new_cfg, n) != s._cfg:
changed_sn.add(n)
# just delete the socket and add it again
deleted_sn.add(n)
added_sn.add(n)
# Get the watchers whichs use these, so they could be
# deleted and added also
for w in self.iter_watchers():
if 'circus.sockets.%s' % n.lower() in w.cmd:
wn_with_changed_socket.add(w.name)
# get deleted sockets
for n in deleted_sn:
s = self.get_socket(n)
s.close()
# Get the watchers whichs use these, these should not be
# active anymore
for w in self.iter_watchers():
if 'circus.sockets.%s' % n.lower() in w.cmd:
wn_with_deleted_socket.add(w.name)
del self.sockets[s.name]
# get added sockets
for n in added_sn:
socket_config = self.get_socket_config(new_cfg, n)
s = CircusSocket.load_from_config(socket_config)
s.bind_and_listen()
self.sockets[s.name] = s
if added_sn or deleted_sn:
# make sure all existing watchers get the new sockets in
# their attributes and get the old removed
# XXX: is this necessary? self.sockets is an mutable
# object
for watcher in self.iter_watchers():
# XXX: What happens as initalize is called on a
# running watcher?
watcher.initialize(self.evpub_socket, self.sockets, self)
# Gather watcher names.
current_wn = set([i.name for i in self.iter_watchers()])
new_wn = set([i['name'] for i in new_cfg.get('watchers', [])])
new_wn = new_wn | set([i['name'] for i in new_cfg.get('plugins', [])])
added_wn = (new_wn - current_wn) | wn_with_changed_socket
deleted_wn = current_wn - new_wn - wn_with_changed_socket
maybechanged_wn = current_wn - deleted_wn
changed_wn = set([])
if wn_with_deleted_socket and wn_with_deleted_socket not in new_wn:
raise ValueError('Watchers %s uses a socket which is deleted' %
wn_with_deleted_socket)
# get changed watchers
for n in maybechanged_wn:
w = self.get_watcher(n)
new_watcher_cfg = (self.get_watcher_config(new_cfg, n) or
self.get_plugin_config(new_cfg, n))
old_watcher_cfg = w._cfg.copy()
if new_watcher_cfg != old_watcher_cfg:
if not w.name.startswith('plugin:'):
num_procs = new_watcher_cfg['numprocesses']
old_watcher_cfg['numprocesses'] = num_procs
if new_watcher_cfg == old_watcher_cfg:
# if nothing but the number of processes is
# changed, just changes this
w.set_numprocesses(int(num_procs))
continue
# Others things are changed. Just delete and add the watcher.
changed_wn.add(n)
deleted_wn.add(n)
added_wn.add(n)
# delete watchers
for n in deleted_wn:
w = self.get_watcher(n)
w.stop()
del self._watchers_names[w.name.lower()]
self.watchers.remove(w)
# add watchers
for n in added_wn:
new_watcher_cfg = (self.get_plugin_config(new_cfg, n) or
self.get_watcher_config(new_cfg, n))
w = Watcher.load_from_config(new_watcher_cfg)
w.initialize(self.evpub_socket, self.sockets, self)
self.start_watcher(w)
self.watchers.append(w)
self._watchers_names[w.name.lower()] = w
return False
def test_load_from_config_no_proto(self):
"""When no proto in the config, the default (0) is used."""
config = {'name': ''}
sock = CircusSocket.load_from_config(config)
self.assertEqual(sock.proto, 0)
sock.close()
def reload_from_config(self, config_file=None, inside_circusd=False):
new_cfg = get_config(config_file if config_file else self.config_file)
# if arbiter is changed, reload everything
if self.get_arbiter_config(new_cfg) != self._cfg:
yield self._restart(inside_circusd=inside_circusd)
return
ignore_sn = set(['circushttpd'])
ignore_wn = set(['circushttpd', 'circusd-stats'])
# Gather socket names.
current_sn = set([i.name for i in self.sockets.values()]) - ignore_sn
new_sn = set([i['name'] for i in new_cfg.get('sockets', [])])
added_sn = new_sn - current_sn
deleted_sn = current_sn - new_sn
maybechanged_sn = current_sn - deleted_sn
changed_sn = set([])
wn_with_changed_socket = set([])
wn_with_deleted_socket = set([])
# get changed sockets
for n in maybechanged_sn:
s = self.get_socket(n)
if self.get_socket_config(new_cfg, n) != s._cfg:
changed_sn.add(n)
# just delete the socket and add it again
deleted_sn.add(n)
added_sn.add(n)
# Get the watchers whichs use these, so they could be
# deleted and added also
for w in self.iter_watchers():
if 'circus.sockets.%s' % n.lower() in w.cmd:
wn_with_changed_socket.add(w.name)
# get deleted sockets
for n in deleted_sn:
s = self.get_socket(n)
s.close()
# Get the watchers whichs use these, these should not be
# active anymore
for w in self.iter_watchers():
if 'circus.sockets.%s' % n.lower() in w.cmd:
wn_with_deleted_socket.add(w.name)
del self.sockets[s.name]
# get added sockets
for n in added_sn:
socket_config = self.get_socket_config(new_cfg, n)
s = CircusSocket.load_from_config(socket_config)
s.bind_and_listen()
self.sockets[s.name] = s
if added_sn or deleted_sn:
# make sure all existing watchers get the new sockets in
# their attributes and get the old removed
# XXX: is this necessary? self.sockets is an mutable
# object
for watcher in self.iter_watchers():
# XXX: What happens as initalize is called on a
# running watcher?
watcher.initialize(self.evpub_socket, self.sockets, self)
# Gather watcher names.
current_wn = set([i.name for i in self.iter_watchers()]) - ignore_wn
new_wn = set([i['name'] for i in new_cfg.get('watchers', [])])
new_wn = new_wn | set([i['name'] for i in new_cfg.get('plugins', [])])
added_wn = (new_wn - current_wn) | wn_with_changed_socket
deleted_wn = current_wn - new_wn - wn_with_changed_socket
maybechanged_wn = current_wn - deleted_wn
changed_wn = set([])
if wn_with_deleted_socket and wn_with_deleted_socket not in new_wn:
raise ValueError('Watchers %s uses a socket which is deleted' %
wn_with_deleted_socket)
# get changed watchers
for n in maybechanged_wn:
w = self.get_watcher(n)
new_watcher_cfg = (self.get_watcher_config(new_cfg, n) or
self.get_plugin_config(new_cfg, n))
old_watcher_cfg = w._cfg.copy()
if 'env' in new_watcher_cfg:
new_watcher_cfg['env'] = parse_env_dict(new_watcher_cfg['env'])
# discarding env exceptions
for key in _ENV_EXCEPTIONS:
if 'env' in new_watcher_cfg and key in new_watcher_cfg['env']:
del new_watcher_cfg['env'][key]
if 'env' in new_watcher_cfg and key in old_watcher_cfg['env']:
del old_watcher_cfg['env'][key]
diff = DictDiffer(new_watcher_cfg, old_watcher_cfg).changed()
if diff == set(['numprocesses']):
# if nothing but the number of processes is
# changed, just changes this
w.set_numprocesses(int(new_watcher_cfg['numprocesses']))
changed = False
else:
changed = len(diff) > 0
if changed:
# Others things are changed. Just delete and add the watcher.
changed_wn.add(n)
deleted_wn.add(n)
added_wn.add(n)
# delete watchers
for n in deleted_wn:
w = self.get_watcher(n)
yield w._stop()
del self._watchers_names[w.name.lower()]
self.watchers.remove(w)
# add watchers
for n in added_wn:
new_watcher_cfg = (self.get_plugin_config(new_cfg, n) or
self.get_watcher_config(new_cfg, n))
w = Watcher.load_from_config(new_watcher_cfg)
w.initialize(self.evpub_socket, self.sockets, self)
yield self.start_watcher(w)
self.watchers.append(w)
self._watchers_names[w.name.lower()] = w
def test_load_from_config_unknown_proto(self):
"""Unknown proto in the config raises an error."""
config = {'name': '', 'proto': 'foo'}
with self.assertRaises(socket.error):
sock = CircusSocket.load_from_config(config)
def reload_from_config(self, config_file=None, inside_circusd=False):
new_cfg = get_config(config_file if config_file else self.config_file)
# if arbiter is changed, reload everything
if self.get_arbiter_config(new_cfg) != self._cfg:
yield self._restart(inside_circusd=inside_circusd)
return
ignore_sn = set(['circushttpd'])
ignore_wn = set(['circushttpd', 'circusd-stats'])
# Gather socket names.
current_sn = set([i.name for i in self.sockets.values()]) - ignore_sn
new_sn = set([i['name'] for i in new_cfg.get('sockets', [])])
added_sn = new_sn - current_sn
deleted_sn = current_sn - new_sn
maybechanged_sn = current_sn - deleted_sn
changed_sn = set([])
wn_with_changed_socket = set([])
wn_with_deleted_socket = set([])
# get changed sockets
for n in maybechanged_sn:
s = self.get_socket(n)
if self.get_socket_config(new_cfg, n) != s._cfg:
changed_sn.add(n)
# just delete the socket and add it again
deleted_sn.add(n)
added_sn.add(n)
# Get the watchers whichs use these, so they could be
# deleted and added also
for w in self.iter_watchers():
if 'circus.sockets.%s' % n.lower() in w.cmd:
wn_with_changed_socket.add(w.name)
# get deleted sockets
for n in deleted_sn:
s = self.get_socket(n)
s.close()
# Get the watchers whichs use these, these should not be
# active anymore
for w in self.iter_watchers():
if 'circus.sockets.%s' % n.lower() in w.cmd:
wn_with_deleted_socket.add(w.name)
del self.sockets[s.name]
# get added sockets
for n in added_sn:
socket_config = self.get_socket_config(new_cfg, n)
s = CircusSocket.load_from_config(socket_config)
s.bind_and_listen()
self.sockets[s.name] = s
if added_sn or deleted_sn:
# make sure all existing watchers get the new sockets in
# their attributes and get the old removed
# XXX: is this necessary? self.sockets is an mutable
# object
for watcher in self.iter_watchers():
# XXX: What happens as initalize is called on a
# running watcher?
watcher.initialize(self.evpub_socket, self.sockets, self)
# Gather watcher names.
current_wn = set([i.name for i in self.iter_watchers()]) - ignore_wn
new_wn = set([i['name'] for i in new_cfg.get('watchers', [])])
new_wn = new_wn | set([i['name'] for i in new_cfg.get('plugins', [])])
added_wn = (new_wn - current_wn) | wn_with_changed_socket
deleted_wn = current_wn - new_wn - wn_with_changed_socket
maybechanged_wn = current_wn - deleted_wn
changed_wn = set([])
if wn_with_deleted_socket and wn_with_deleted_socket not in new_wn:
raise ValueError('Watchers %s uses a socket which is deleted' %
wn_with_deleted_socket)
# get changed watchers
for n in maybechanged_wn:
w = self.get_watcher(n)
new_watcher_cfg = (self.get_watcher_config(new_cfg, n)
or self.get_plugin_config(new_cfg, n))
old_watcher_cfg = w._cfg.copy()
if 'env' in new_watcher_cfg:
new_watcher_cfg['env'] = parse_env_dict(new_watcher_cfg['env'])
# discarding env exceptions
for key in _ENV_EXCEPTIONS:
if 'env' in new_watcher_cfg and key in new_watcher_cfg['env']:
del new_watcher_cfg['env'][key]
if 'env' in new_watcher_cfg and key in old_watcher_cfg['env']:
del old_watcher_cfg['env'][key]
diff = DictDiffer(new_watcher_cfg, old_watcher_cfg).changed()
if diff == set(['numprocesses']):
# if nothing but the number of processes is
# changed, just changes this
w.set_numprocesses(int(new_watcher_cfg['numprocesses']))
changed = False
else:
changed = len(diff) > 0
if changed:
# Others things are changed. Just delete and add the watcher.
changed_wn.add(n)
deleted_wn.add(n)
added_wn.add(n)
# delete watchers
for n in deleted_wn:
w = self.get_watcher(n)
yield w._stop()
del self._watchers_names[w.name.lower()]
self.watchers.remove(w)
# add watchers
for n in added_wn:
new_watcher_cfg = (self.get_plugin_config(new_cfg, n)
or self.get_watcher_config(new_cfg, n))
w = Watcher.load_from_config(new_watcher_cfg)
w.initialize(self.evpub_socket, self.sockets, self)
yield self.start_watcher(w)
self.watchers.append(w)
self._watchers_names[w.name.lower()] = w
def reload_from_config(self, config_file=None):
cfg = get_config(config_file if config_file else self.config_file)
# if arbiter is changed, reload everything
if self.cfg2dict(cfg) != self.cfg:
return True
current_socket_names = set([i.name for i in self.sockets])
new_socket_names = set([i['name'] for i in cfg.get('sockets', [])])
added_socket_names = new_socket_names - current_socket_names
deleted_socket_names = current_socket_names - new_socket_names
maybechanged_socket_names = current_socket_names - deleted_socket_names
changed_socket_names = set([])
watcher_names_with_changed_socket = set([])
watcher_names_with_deleted_socket = set([])
# get changed sockets
for n in maybechanged_socket_names:
s = self.get_socket(n)
if s.cfg2dict(self.get_socket_config(cfg, n)) != s.cfg:
changed_socket_names.add(n)
# just delete the socket and add it again
deleted_socket_names.add(n)
added_socket_names.add(n)
# Get the watchers whichs use these, so they could be deleted and added also
for w in self.iter_watchers:
if 'circus.sockets.%s' % n.lower() in w.cmd:
watcher_names_with_changed_socket.add(w.name)
# get deleted sockets
for n in deleted_socket_names:
s = self.get_socket(n)
s.close()
# Get the watchers whichs use these, these should not be active anymore
for w in self.iter_watchers():
if 'circus.sockets.%s' % n.lower() in w.cmd:
watcher_names_with_deleted_socket.add(w.name)
self.sockets.remove(s)
# get added sockets
for n in added_socket_names:
s = CircusSocket.load_from_config(self.get_socket_config(cfg, n))
s.bind_and_listen()
self.sockets.append(s)
if added_socket_names or deleted_socket_names:
# make sure all existing watchers get the new sockets in their attributes and get the old removed
for watcher in self.iter_watchers():
watcher.initialize(self.evpub_socket, self.sockets, self)
current_watcher_names = set([i.name for i in self.iter_watchers()])
new_watcher_names = set([i['name'] for i in cfg.get('watchers', [])])
added_watcher_names = (new_watcher_names - current_watcher_names) | watcher_names_with_changed_socket
deleted_watcher_names = current_watcher_names - new_watcher_names - watcher_names_with_changed_socket
maybechanged_watcher_names = current_watcher_names - deleted_watcher_names
changed_watcher_names = set([])
if watcher_names_with_deleted_socket and watcher_names_with_deleted_socket not in new_watcher_names:
raise ValueError('Watchers %s uses a socket which is deleted' % watcher_names_with_deleted_socket)
#get changed watchers
for n in maybechanged_watcher_names:
w = self.get_watcher(n)
new_cfg = w.cfg2dict(self.get_watcher_config(cfg, n))
old_cfg = w.cfg2dict(w.cfg) # cfg2dict is used to make sure a copy is returned
if new_cfg != old_cfg:
old_cfg['numprocesses'] = new_cfg['numprocesses']
if new_cfg == old_cfg:
# if nothing but the number of processes is changed, just changes this
w.set_numprocesses(int(new_cfg['numprocesses']))
else:
# Other thing are changed. Just delete and add the watcher.
changed_watcher_names.add(n)
deleted_watcher_names.add(n)
added_watcher_names.add(n)
# get deleted watchers
for n in deleted_watcher_names:
w = self.get_watcher(n)
w.stop()
del self._watchers_names[w.name.lower()]
self.watchers.remove(w)
# get added watchers
for n in added_watcher_names:
w = Watcher.load_from_config(self.get_watcher_config(cfg, n))
w.initialize(self.evpub_socket, self.sockets, self)
w.start()
self.watchers.append(w)
self._watchers_names[w.name.lower()] = w
return False
def test_load_from_config_unknown_proto(self):
"""Unknown proto in the config raises an error."""
config = {'name': '', 'proto': 'foo'}
with self.assertRaises(socket.error):
sock = CircusSocket.load_from_config(config)
