def load_from_config(cls, config_file): cfg = get_config(config_file) # hack reload ioloop to use the monkey patched version reload(ioloop) watchers = [] for watcher in cfg.get('watchers', []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket in cfg.get('sockets', []): sockets.append(CircusSocket.load_from_config(socket)) # creating arbiter arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'], check_delay=cfg.get('check_delay', 1.), prereload_fn=cfg.get('prereload_fn'), stats_endpoint=cfg.get('stats_endpoint'), plugins=cfg.get('plugins'), sockets=sockets, warmup_delay=cfg.get('warmup_delay', 0), httpd=cfg.get('httpd', False), httpd_host=cfg.get('httpd_host', 'localhost'), httpd_port=cfg.get('httpd_port', 8080)) return arbiter
def load_from_config(cls, config_file): cfg = get_config(config_file) # hack reload ioloop to use the monkey patched version reload(ioloop) watchers = [] for watcher in cfg.get('watchers', []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket in cfg.get('sockets', []): sockets.append(CircusSocket.load_from_config(socket)) # creating arbiter arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'], check_delay=cfg.get('check_delay', 1.), prereload_fn=cfg.get('prereload_fn'), stats_endpoint=cfg.get('stats_endpoint'), plugins=cfg.get('plugins'), sockets=sockets, warmup_delay=cfg.get('warmup_delay', 0), httpd=cfg.get('httpd', False), httpd_host=cfg.get('httpd_host', 'localhost'), httpd_port=cfg.get('httpd_port', 8080), debug=cfg.get('debug', False), stream_backend=cfg.get('stream_backend', 'thread'), ssh_server=cfg.get('ssh_server', None)) return arbiter
def load_from_config(cls, config_file): cfg = get_config(config_file) watchers = [] for watcher in cfg.get('watchers', []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket in cfg.get('sockets', []): sockets.append(CircusSocket.load_from_config(socket)) httpd = cfg.get('httpd', False) if httpd: # controlling that we have what it takes to run the web UI # if something is missing this will tell the user try: import circusweb # NOQA except ImportError: logger.error('You need to install circus-web') sys.exit(1) # creating arbiter arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'], check_delay=cfg.get('check_delay', 1.), prereload_fn=cfg.get('prereload_fn'), stats_endpoint=cfg.get('stats_endpoint'), plugins=cfg.get('plugins'), sockets=sockets, warmup_delay=cfg.get('warmup_delay', 0), httpd=httpd, httpd_host=cfg.get('httpd_host', 'localhost'), httpd_port=cfg.get('httpd_port', 8080), debug=cfg.get('debug', False), ssh_server=cfg.get('ssh_server', None)) return arbiter
def load_from_config(cls, config_file): cfg = get_config(config_file) # hack reload ioloop to use the monkey patched version reload(ioloop) watchers = [] for watcher in cfg.get("watchers", []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket in cfg.get("sockets", []): sockets.append(CircusSocket.load_from_config(socket)) # creating arbiter arbiter = cls( watchers, cfg["endpoint"], cfg["pubsub_endpoint"], check_delay=cfg.get("check_delay", 1.0), prereload_fn=cfg.get("prereload_fn"), stats_endpoint=cfg.get("stats_endpoint"), plugins=cfg.get("plugins"), sockets=sockets, ) return arbiter
def _get_sockets_fds(self): """Returns sockets dict. If this worker's cmd indicates use of a SO_REUSEPORT socket, a new socket is created and bound. This new socket's FD replaces original socket's FD in returned dict. This method populates `self._sockets` list. This list should be let go after `fork()`. """ sockets_fds = None if self.watcher is not None and self.watcher.sockets is not None: sockets_fds = self.watcher._get_sockets_fds() reuseport_sockets = tuple((sn, s) for (sn, s) in self.watcher.sockets.items() if s.so_reuseport) for sn, s in reuseport_sockets: # watcher.cmd uses this reuseport socket if 'circus.sockets.%s' % sn in self.watcher.cmd: sock = CircusSocket.load_from_config(s._cfg) sock.bind_and_listen() # replace original socket's fd sockets_fds[sn] = sock.fileno() # keep new socket until fork returns self._sockets.append(sock) return sockets_fds
def _get_sockets_fds(self): """Returns sockets dict. If this worker's cmd indicates use of a SO_REUSEPORT socket, a new socket is created and bound. This new socket's FD replaces original socket's FD in returned dict. This method populates `self._sockets` list. This list should be let go after `fork()`. """ sockets_fds = None if self.watcher is not None and self.watcher.sockets is not None: sockets_fds = self.watcher._get_sockets_fds() reuseport_sockets = tuple( (sn, s) for (sn, s) in self.watcher.sockets.items() if s.so_reuseport) for sn, s in reuseport_sockets: # watcher.cmd uses this reuseport socket if 'circus.sockets.%s' % sn in self.watcher.cmd: sock = CircusSocket.load_from_config(s._cfg) sock.bind_and_listen() # replace original socket's fd sockets_fds[sn] = sock.fileno() # keep new socket until fork returns self._sockets.append(sock) return sockets_fds
def test_issue310(self): """ https://github.com/mozilla-services/circus/pull/310 Allow $(circus.sockets.name) to be used in args. """ conf = get_config(_CONF["issue310"]) watcher = Watcher.load_from_config(conf["watchers"][0]) socket = CircusSocket.load_from_config(conf["sockets"][0]) watcher.initialize(None, {"web": socket}, None) process = Process( watcher._process_counter, watcher.cmd, args=watcher.args, working_dir=watcher.working_dir, shell=watcher.shell, uid=watcher.uid, gid=watcher.gid, env=watcher.env, rlimits=watcher.rlimits, spawn=False, executable=watcher.executable, use_fds=watcher.use_sockets, watcher=watcher, ) fd = watcher._get_sockets_fds()["web"] formatted_args = process.format_args() self.assertEquals(formatted_args, ["foo", "--fd", str(fd)])
def load_from_config(cls, config_file): cfg = get_config(config_file) # hack reload ioloop to use the monkey patched version reload(ioloop) watchers = [] for watcher in cfg.get('watchers', []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket in cfg.get('sockets', []): sockets.append(CircusSocket.load_from_config(socket)) # creating arbiter arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'], check_delay=cfg.get('check_delay', 1.), prereload_fn=cfg.get('prereload_fn'), stats_endpoint=cfg.get('stats_endpoint'), plugins=cfg.get('plugins'), sockets=sockets) return arbiter
def test_issue310(self): ''' https://github.com/mozilla-services/circus/pull/310 Allow $(circus.sockets.name) to be used in args. ''' conf = get_config(_CONF['issue310']) watcher = Watcher.load_from_config(conf['watchers'][0]) socket = CircusSocket.load_from_config(conf['sockets'][0]) try: watcher.initialize(None, {'web': socket}, None) process = Process(watcher._nextwid, watcher.cmd, args=watcher.args, working_dir=watcher.working_dir, shell=watcher.shell, uid=watcher.uid, gid=watcher.gid, env=watcher.env, rlimits=watcher.rlimits, spawn=False, executable=watcher.executable, use_fds=watcher.use_sockets, watcher=watcher) sockets_fds = watcher._get_sockets_fds() formatted_args = process.format_args(sockets_fds=sockets_fds) fd = sockets_fds['web'] self.assertEqual(formatted_args, ['foo', '--fd', str(fd)]) finally: socket.close()
def load_from_config(cls, config_file): cfg = get_config(config_file) watchers = [] for watcher in cfg.get('watchers', []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket in cfg.get('sockets', []): sockets.append(CircusSocket.load_from_config(socket)) # creating arbiter arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'], check_delay=cfg.get('check_delay', 1.), prereload_fn=cfg.get('prereload_fn'), stats_endpoint=cfg.get('stats_endpoint'), plugins=cfg.get('plugins'), sockets=sockets, warmup_delay=cfg.get('warmup_delay', 0), httpd=cfg.get('httpd', False), httpd_host=cfg.get('httpd_host', 'localhost'), httpd_port=cfg.get('httpd_port', 8080), debug=cfg.get('debug', False), stream_backend=cfg.get('stream_backend', 'thread'), ssh_server=cfg.get('ssh_server', None)) return arbiter
def load_from_config(cls, config_file): cfg = get_config(config_file) watchers = [] for watcher in cfg.get('watchers', []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket in cfg.get('sockets', []): sockets.append(CircusSocket.load_from_config(socket)) httpd = cfg.get('httpd', False) if httpd: # controlling that we have what it takes to run the web UI # if something is missing this will tell the user try: import circusweb # NOQA except ImportError: logger.error('You need to install circus-web') sys.exit(1) # creating arbiter arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'], check_delay=cfg.get('check_delay', 1.), prereload_fn=cfg.get('prereload_fn'), stats_endpoint=cfg.get('stats_endpoint'), plugins=cfg.get('plugins'), sockets=sockets, warmup_delay=cfg.get('warmup_delay', 0), httpd=httpd, httpd_host=cfg.get('httpd_host', 'localhost'), httpd_port=cfg.get('httpd_port', 8080), debug=cfg.get('debug', False), ssh_server=cfg.get('ssh_server', None)) return arbiter
def test_load_from_config_replace(self): sockfile = self._get_file() config = {'name': 'somename', 'path': sockfile, 'replace': False} sock = CircusSocket.load_from_config(config) try: self.assertRaises(OSError, sock.bind_and_listen) finally: sock.close() config = {'name': 'somename', 'path': sockfile, 'replace': True} sock = CircusSocket.load_from_config(config) sock.bind_and_listen() try: self.assertEqual(sock.replace, True) finally: sock.close()
def test_load_from_config_umask(self): sockfile = self._get_tmp_filename() config = {"name": "somename", "path": sockfile, "umask": 0} sock = CircusSocket.load_from_config(config) try: self.assertEqual(sock.umask, 0) finally: sock.close()
def test_load_from_config_umask(self): sockfile = self._get_tmp_filename() config = {'name': 'somename', 'path': sockfile, 'umask': 0} sock = CircusSocket.load_from_config(config) try: self.assertEqual(sock.umask, 0) finally: sock.close()
def test_reuseport_supported(self): config = {"name": "", "host": "localhost", "port": 0, "so_reuseport": True} sock = CircusSocket.load_from_config(config) sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) self.assertEqual(sock.so_reuseport, True) self.assertNotEqual(sockopt, 0)
def test_load_from_config_blocking(self): # test default to false config = {'name': 'somename', 'host': 'localhost', 'port': 0} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.blocking, False) sock.bind_and_listen() self.assertTrue(is_nonblock(sock.fileno())) sock.close() # test when true config = {'name': 'somename', 'host': 'localhost', 'port': 0, 'blocking': True} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.blocking, True) sock.bind_and_listen() self.assertFalse(is_nonblock(sock.fileno())) sock.close()
def test_load_from_config_umask(self): sockfile = self._get_tmp_filename() config = {'name': 'somename', 'path': sockfile, 'umask': 0} sock = CircusSocket.load_from_config(config) try: self.assertEqual(sock.umask, 0) finally: sock.close()
def test_load_from_config_replace(self): sockfile = self._get_file() config = {'name': 'somename', 'path': sockfile, 'replace': False} sock = CircusSocket.load_from_config(config) try: self.assertRaises(OSError, sock.bind_and_listen) finally: sock.close() config = {'name': 'somename', 'path': sockfile, 'replace': True} sock = CircusSocket.load_from_config(config) sock.bind_and_listen() try: self.assertEqual(sock.replace, True) finally: sock.close()
def test_reuseport_supported(self): config = {'name': '', 'host': 'localhost', 'port': 0, 'so_reuseport': True} sock = CircusSocket.load_from_config(config) sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) self.assertEqual(sock.so_reuseport, True) self.assertNotEqual(sockopt, 0)
def test_load_from_config_blocking(self): # test default to false config = {'name': 'somename', 'host': 'localhost', 'port': 0} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.blocking, False) sock.bind_and_listen() fl = fcntl.fcntl(sock.fileno(), fcntl.F_GETFL) self.assertTrue(is_nonblock(sock.fileno())) sock.close() # test when true config = {'name': 'somename', 'host': 'localhost', 'port': 0, 'blocking': True} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.blocking, True) sock.bind_and_listen() fl2 = fcntl.fcntl(sock.fileno(), fcntl.F_GETFL) self.assertFalse(is_nonblock(sock.fileno())) sock.close()
def test_bind_to_interface(self): config = {"name": "", "host": "localhost", "port": 0, "interface": "lo"} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.interface, config["interface"]) sock.setsockopt = mock.Mock() try: sock.bind_and_listen() sock.setsockopt.assert_any_call(socket.SOL_SOCKET, IN.SO_BINDTODEVICE, config["interface"] + "\0") finally: sock.close()
def test_load_from_config_umask(self): fd, sockfile = tempfile.mkstemp() os.close(fd) os.remove(sockfile) config = {"name": "somename", "path": sockfile, "umask": 0} sock = CircusSocket.load_from_config(config) try: self.assertEqual(sock.umask, 0) finally: sock.close()
def load_from_config(cls, config_file, loop=None): cfg = get_config(config_file) watchers = [] for watcher in cfg.get('watchers', []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket_ in cfg.get('sockets', []): sockets.append(CircusSocket.load_from_config(socket_)) httpd = cfg.get('httpd', False) if httpd: # controlling that we have what it takes to run the web UI # if something is missing this will tell the user try: import circusweb # NOQA except ImportError: logger.error('You need to install circus-web') sys.exit(1) # creating arbiter arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'], check_delay=cfg.get('check_delay', 1.), prereload_fn=cfg.get('prereload_fn'), statsd=cfg.get('statsd', False), stats_endpoint=cfg.get('stats_endpoint'), papa_endpoint=cfg.get('papa_endpoint'), multicast_endpoint=cfg.get('multicast_endpoint'), plugins=cfg.get('plugins'), sockets=sockets, warmup_delay=cfg.get('warmup_delay', 0), httpd=httpd, loop=loop, httpd_host=cfg.get('httpd_host', 'localhost'), httpd_port=cfg.get('httpd_port', 8080), debug=cfg.get('debug', False), debug_gc=cfg.get('debug_gc', False), ssh_server=cfg.get('ssh_server', None), pidfile=cfg.get('pidfile', None), loglevel=cfg.get('loglevel', None), logoutput=cfg.get('logoutput', None), loggerconfig=cfg.get('loggerconfig', None), fqdn_prefix=cfg.get('fqdn_prefix', None), umask=cfg['umask'], endpoint_owner=cfg.get('endpoint_owner', None)) # store the cfg which will be used, so it can be used later # for checking if the cfg has been changed arbiter._cfg = cls.get_arbiter_config(cfg) arbiter.config_file = config_file return arbiter
def test_load_from_config_umask(self): fd, sockfile = tempfile.mkstemp() os.close(fd) os.remove(sockfile) config = {'name': 'somename', 'path': sockfile, 'umask': 0} sock = CircusSocket.load_from_config(config) try: self.assertEqual(sock.umask, 0) finally: sock.close()
def test_load_from_config_umask(self): fd, sockfile = tempfile.mkstemp() os.close(fd) os.remove(sockfile) config = {'name': 'somename', 'path': sockfile, 'umask': 0} sock = CircusSocket.load_from_config(config) try: self.assertEqual(sock.umask, 0) finally: sock.close()
def test_inet6(self): config = {'name': '', 'host': '::1', 'port': 0, 'family': 'AF_INET6'} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.host, config['host']) self.assertEqual(sock.port, config['port']) sock.setsockopt = mock.Mock() try: sock.bind_and_listen() # we should have got a port set self.assertNotEqual(sock.port, 0) finally: sock.close()
def test_inet6(self): config = {"name": "", "host": "::1", "port": 0, "family": "AF_INET6"} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.host, config["host"]) self.assertEqual(sock.port, config["port"]) sock.setsockopt = mock.Mock() try: sock.bind_and_listen() # we should have got a port set self.assertNotEqual(sock.port, 0) finally: sock.close()
def load_from_config(cls, config_file, loop=None): cfg = get_config(config_file) watchers = [] for watcher in cfg.get("watchers", []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket in cfg.get("sockets", []): sockets.append(CircusSocket.load_from_config(socket)) httpd = cfg.get("httpd", False) if httpd: # controlling that we have what it takes to run the web UI # if something is missing this will tell the user try: import circusweb # NOQA except ImportError: logger.error("You need to install circus-web") sys.exit(1) # creating arbiter arbiter = cls( watchers, cfg["endpoint"], cfg["pubsub_endpoint"], check_delay=cfg.get("check_delay", 1.0), prereload_fn=cfg.get("prereload_fn"), statsd=cfg.get("statsd", False), stats_endpoint=cfg.get("stats_endpoint"), multicast_endpoint=cfg.get("multicast_endpoint"), plugins=cfg.get("plugins"), sockets=sockets, warmup_delay=cfg.get("warmup_delay", 0), httpd=httpd, loop=loop, httpd_host=cfg.get("httpd_host", "localhost"), httpd_port=cfg.get("httpd_port", 8080), debug=cfg.get("debug", False), ssh_server=cfg.get("ssh_server", None), pidfile=cfg.get("pidfile", None), loglevel=cfg.get("loglevel", None), logoutput=cfg.get("logoutput", None), fqdn_prefix=cfg.get("fqdn_prefix", None), umask=cfg["umask"], ) # store the cfg which will be used, so it can be used later # for checking if the cfg has been changed arbiter._cfg = cls.get_arbiter_config(cfg) arbiter.config_file = config_file return arbiter
def test_bind_to_interface(self): config = {'name': '', 'host': 'localhost', 'port': 0, 'interface': 'lo'} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.interface, config['interface']) sock.setsockopt = mock.Mock() try: sock.bind_and_listen() sock.setsockopt.assert_any_call(socket.SOL_SOCKET, IN.SO_BINDTODEVICE, config['interface'] + '\0') finally: sock.close()
def test_inet6(self): config = {'name': '', 'host': '::1', 'port': 0, 'family': 'AF_INET6'} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.host, config['host']) self.assertEqual(sock.port, config['port']) sock.setsockopt = mock.Mock() try: sock.bind_and_listen() # we should have got a port set self.assertNotEqual(sock.port, 0) finally: sock.close()
def test_reuseport_unsupported(self): config = {"name": "", "host": "localhost", "port": 0, "so_reuseport": True} saved = None try: if hasattr(socket, "SO_REUSEPORT"): saved = socket.SO_REUSEPORT del socket.SO_REUSEPORT sock = CircusSocket.load_from_config(config) self.assertEqual(sock.so_reuseport, False) finally: if saved is not None: socket.SO_REUSEPORT = saved
def load_from_config(cls, config_file, loop=None): cfg = get_config(config_file) watchers = [] for watcher in cfg.get('watchers', []): watchers.append(Watcher.load_from_config(watcher)) sockets = [] for socket_ in cfg.get('sockets', []): sockets.append(CircusSocket.load_from_config(socket_)) httpd = cfg.get('httpd', False) if httpd: # controlling that we have what it takes to run the web UI # if something is missing this will tell the user try: import circusweb # NOQA except ImportError: logger.error('You need to install circus-web') sys.exit(1) # creating arbiter arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'], check_delay=cfg.get('check_delay', 1.), prereload_fn=cfg.get('prereload_fn'), statsd=cfg.get('statsd', False), stats_endpoint=cfg.get('stats_endpoint'), papa_endpoint=cfg.get('papa_endpoint'), multicast_endpoint=cfg.get('multicast_endpoint'), plugins=cfg.get('plugins'), sockets=sockets, warmup_delay=cfg.get('warmup_delay', 0), httpd=httpd, loop=loop, httpd_host=cfg.get('httpd_host', 'localhost'), httpd_port=cfg.get('httpd_port', 8080), debug=cfg.get('debug', False), debug_gc=cfg.get('debug_gc', False), ssh_server=cfg.get('ssh_server', None), pidfile=cfg.get('pidfile', None), loglevel=cfg.get('loglevel', None), logoutput=cfg.get('logoutput', None), loggerconfig=cfg.get('loggerconfig', None), fqdn_prefix=cfg.get('fqdn_prefix', None), umask=cfg['umask'], endpoint_owner=cfg.get('endpoint_owner', None)) # store the cfg which will be used, so it can be used later # for checking if the cfg has been changed arbiter._cfg = cls.get_arbiter_config(cfg) arbiter.config_file = config_file return arbiter
def test_reuseport_supported(self): config = {"name": "", "host": "localhost", "port": 0, "so_reuseport": True} sock = CircusSocket.load_from_config(config) try: sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) except socket.error: # see #699 return finally: sock.close() self.assertEqual(sock.so_reuseport, True) self.assertNotEqual(sockopt, 0)
def test_reuseport_unsupported(self): config = {'name': '', 'host': 'localhost', 'port': 0, 'so_reuseport': True} saved = None try: if hasattr(socket, 'SO_REUSEPORT'): saved = socket.SO_REUSEPORT del socket.SO_REUSEPORT sock = CircusSocket.load_from_config(config) self.assertEqual(sock.so_reuseport, False) finally: if saved is not None: socket.SO_REUSEPORT = saved
def test_reuseport_unsupported(self): config = {'name': '', 'host': 'localhost', 'port': 0, 'so_reuseport': True} saved = None try: if hasattr(socket, 'SO_REUSEPORT'): saved = socket.SO_REUSEPORT del socket.SO_REUSEPORT sock = CircusSocket.load_from_config(config) self.assertEqual(sock.so_reuseport, False) finally: if saved is not None: socket.SO_REUSEPORT = saved
def test_reuseport_supported(self): config = {'name': '', 'host': 'localhost', 'port': 0, 'so_reuseport': True} sock = CircusSocket.load_from_config(config) try: sockopt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) except socket.error: # see #699 return finally: sock.close() self.assertEqual(sock.so_reuseport, True) self.assertNotEqual(sockopt, 0)
def test_issue310(self): """ https://github.com/mozilla-services/circus/pull/310 Allow $(circus.sockets.name) to be used in args. """ conf = get_config(_CONF["issue310"]) watcher = Watcher.load_from_config(conf["watchers"][0]) socket = CircusSocket.load_from_config(conf["sockets"][0]) try: watcher.initialize(None, {"web": socket}, None) if IS_WINDOWS: # We can't close the sockets on Windows as we # are redirecting stdout watcher.use_sockets = True process = Process( watcher._nextwid, watcher.cmd, args=watcher.args, working_dir=watcher.working_dir, shell=watcher.shell, uid=watcher.uid, gid=watcher.gid, env=watcher.env, rlimits=watcher.rlimits, spawn=False, executable=watcher.executable, use_fds=watcher.use_sockets, watcher=watcher, ) sockets_fds = watcher._get_sockets_fds() formatted_args = process.format_args(sockets_fds=sockets_fds) fd = sockets_fds["web"] self.assertEqual(formatted_args, ["foo", "--fd", str(fd)]) finally: socket.close()
def test_load_from_config_no_proto(self): """When no proto in the config, the default (0) is used.""" config = {'name': ''} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.proto, 0)
def reload_from_config(self, config_file=None): new_cfg = get_config(config_file if config_file else self.config_file) # if arbiter is changed, reload everything if self.get_arbiter_config(new_cfg) != self._cfg: raise ReloadArbiterException # Gather socket names. current_sn = set([i.name for i in self.sockets.values()]) new_sn = set([i['name'] for i in new_cfg.get('sockets', [])]) added_sn = new_sn - current_sn deleted_sn = current_sn - new_sn maybechanged_sn = current_sn - deleted_sn changed_sn = set([]) wn_with_changed_socket = set([]) wn_with_deleted_socket = set([]) # get changed sockets for n in maybechanged_sn: s = self.get_socket(n) if self.get_socket_config(new_cfg, n) != s._cfg: changed_sn.add(n) # just delete the socket and add it again deleted_sn.add(n) added_sn.add(n) # Get the watchers whichs use these, so they could be # deleted and added also for w in self.iter_watchers(): if 'circus.sockets.%s' % n.lower() in w.cmd: wn_with_changed_socket.add(w.name) # get deleted sockets for n in deleted_sn: s = self.get_socket(n) s.close() # Get the watchers whichs use these, these should not be # active anymore for w in self.iter_watchers(): if 'circus.sockets.%s' % n.lower() in w.cmd: wn_with_deleted_socket.add(w.name) del self.sockets[s.name] # get added sockets for n in added_sn: socket_config = self.get_socket_config(new_cfg, n) s = CircusSocket.load_from_config(socket_config) s.bind_and_listen() self.sockets[s.name] = s if added_sn or deleted_sn: # make sure all existing watchers get the new sockets in # their attributes and get the old removed # XXX: is this necessary? self.sockets is an mutable # object for watcher in self.iter_watchers(): # XXX: What happens as initalize is called on a # running watcher? watcher.initialize(self.evpub_socket, self.sockets, self) # Gather watcher names. current_wn = set([i.name for i in self.iter_watchers()]) new_wn = set([i['name'] for i in new_cfg.get('watchers', [])]) new_wn = new_wn | set([i['name'] for i in new_cfg.get('plugins', [])]) added_wn = (new_wn - current_wn) | wn_with_changed_socket deleted_wn = current_wn - new_wn - wn_with_changed_socket maybechanged_wn = current_wn - deleted_wn changed_wn = set([]) if wn_with_deleted_socket and wn_with_deleted_socket not in new_wn: raise ValueError('Watchers %s uses a socket which is deleted' % wn_with_deleted_socket) # get changed watchers for n in maybechanged_wn: w = self.get_watcher(n) new_watcher_cfg = (self.get_watcher_config(new_cfg, n) or self.get_plugin_config(new_cfg, n)) old_watcher_cfg = w._cfg.copy() if new_watcher_cfg != old_watcher_cfg: if not w.name.startswith('plugin:'): num_procs = new_watcher_cfg['numprocesses'] old_watcher_cfg['numprocesses'] = num_procs if new_watcher_cfg == old_watcher_cfg: # if nothing but the number of processes is # changed, just changes this w.set_numprocesses(int(num_procs)) continue # Others things are changed. Just delete and add the watcher. changed_wn.add(n) deleted_wn.add(n) added_wn.add(n) # delete watchers for n in deleted_wn: w = self.get_watcher(n) w.stop() del self._watchers_names[w.name.lower()] self.watchers.remove(w) # add watchers for n in added_wn: new_watcher_cfg = (self.get_plugin_config(new_cfg, n) or self.get_watcher_config(new_cfg, n)) w = Watcher.load_from_config(new_watcher_cfg) w.initialize(self.evpub_socket, self.sockets, self) self.start_watcher(w) self.watchers.append(w) self._watchers_names[w.name.lower()] = w return False
def test_load_from_config_no_proto(self): """When no proto in the config, the default (0) is used.""" config = {'name': ''} sock = CircusSocket.load_from_config(config) self.assertEqual(sock.proto, 0) sock.close()
def reload_from_config(self, config_file=None, inside_circusd=False): new_cfg = get_config(config_file if config_file else self.config_file) # if arbiter is changed, reload everything if self.get_arbiter_config(new_cfg) != self._cfg: yield self._restart(inside_circusd=inside_circusd) return ignore_sn = set(['circushttpd']) ignore_wn = set(['circushttpd', 'circusd-stats']) # Gather socket names. current_sn = set([i.name for i in self.sockets.values()]) - ignore_sn new_sn = set([i['name'] for i in new_cfg.get('sockets', [])]) added_sn = new_sn - current_sn deleted_sn = current_sn - new_sn maybechanged_sn = current_sn - deleted_sn changed_sn = set([]) wn_with_changed_socket = set([]) wn_with_deleted_socket = set([]) # get changed sockets for n in maybechanged_sn: s = self.get_socket(n) if self.get_socket_config(new_cfg, n) != s._cfg: changed_sn.add(n) # just delete the socket and add it again deleted_sn.add(n) added_sn.add(n) # Get the watchers whichs use these, so they could be # deleted and added also for w in self.iter_watchers(): if 'circus.sockets.%s' % n.lower() in w.cmd: wn_with_changed_socket.add(w.name) # get deleted sockets for n in deleted_sn: s = self.get_socket(n) s.close() # Get the watchers whichs use these, these should not be # active anymore for w in self.iter_watchers(): if 'circus.sockets.%s' % n.lower() in w.cmd: wn_with_deleted_socket.add(w.name) del self.sockets[s.name] # get added sockets for n in added_sn: socket_config = self.get_socket_config(new_cfg, n) s = CircusSocket.load_from_config(socket_config) s.bind_and_listen() self.sockets[s.name] = s if added_sn or deleted_sn: # make sure all existing watchers get the new sockets in # their attributes and get the old removed # XXX: is this necessary? self.sockets is an mutable # object for watcher in self.iter_watchers(): # XXX: What happens as initalize is called on a # running watcher? watcher.initialize(self.evpub_socket, self.sockets, self) # Gather watcher names. current_wn = set([i.name for i in self.iter_watchers()]) - ignore_wn new_wn = set([i['name'] for i in new_cfg.get('watchers', [])]) new_wn = new_wn | set([i['name'] for i in new_cfg.get('plugins', [])]) added_wn = (new_wn - current_wn) | wn_with_changed_socket deleted_wn = current_wn - new_wn - wn_with_changed_socket maybechanged_wn = current_wn - deleted_wn changed_wn = set([]) if wn_with_deleted_socket and wn_with_deleted_socket not in new_wn: raise ValueError('Watchers %s uses a socket which is deleted' % wn_with_deleted_socket) # get changed watchers for n in maybechanged_wn: w = self.get_watcher(n) new_watcher_cfg = (self.get_watcher_config(new_cfg, n) or self.get_plugin_config(new_cfg, n)) old_watcher_cfg = w._cfg.copy() if 'env' in new_watcher_cfg: new_watcher_cfg['env'] = parse_env_dict(new_watcher_cfg['env']) # discarding env exceptions for key in _ENV_EXCEPTIONS: if 'env' in new_watcher_cfg and key in new_watcher_cfg['env']: del new_watcher_cfg['env'][key] if 'env' in new_watcher_cfg and key in old_watcher_cfg['env']: del old_watcher_cfg['env'][key] diff = DictDiffer(new_watcher_cfg, old_watcher_cfg).changed() if diff == set(['numprocesses']): # if nothing but the number of processes is # changed, just changes this w.set_numprocesses(int(new_watcher_cfg['numprocesses'])) changed = False else: changed = len(diff) > 0 if changed: # Others things are changed. Just delete and add the watcher. changed_wn.add(n) deleted_wn.add(n) added_wn.add(n) # delete watchers for n in deleted_wn: w = self.get_watcher(n) yield w._stop() del self._watchers_names[w.name.lower()] self.watchers.remove(w) # add watchers for n in added_wn: new_watcher_cfg = (self.get_plugin_config(new_cfg, n) or self.get_watcher_config(new_cfg, n)) w = Watcher.load_from_config(new_watcher_cfg) w.initialize(self.evpub_socket, self.sockets, self) yield self.start_watcher(w) self.watchers.append(w) self._watchers_names[w.name.lower()] = w
def test_load_from_config_unknown_proto(self): """Unknown proto in the config raises an error.""" config = {'name': '', 'proto': 'foo'} with self.assertRaises(socket.error): sock = CircusSocket.load_from_config(config)
def reload_from_config(self, config_file=None, inside_circusd=False): new_cfg = get_config(config_file if config_file else self.config_file) # if arbiter is changed, reload everything if self.get_arbiter_config(new_cfg) != self._cfg: yield self._restart(inside_circusd=inside_circusd) return ignore_sn = set(['circushttpd']) ignore_wn = set(['circushttpd', 'circusd-stats']) # Gather socket names. current_sn = set([i.name for i in self.sockets.values()]) - ignore_sn new_sn = set([i['name'] for i in new_cfg.get('sockets', [])]) added_sn = new_sn - current_sn deleted_sn = current_sn - new_sn maybechanged_sn = current_sn - deleted_sn changed_sn = set([]) wn_with_changed_socket = set([]) wn_with_deleted_socket = set([]) # get changed sockets for n in maybechanged_sn: s = self.get_socket(n) if self.get_socket_config(new_cfg, n) != s._cfg: changed_sn.add(n) # just delete the socket and add it again deleted_sn.add(n) added_sn.add(n) # Get the watchers whichs use these, so they could be # deleted and added also for w in self.iter_watchers(): if 'circus.sockets.%s' % n.lower() in w.cmd: wn_with_changed_socket.add(w.name) # get deleted sockets for n in deleted_sn: s = self.get_socket(n) s.close() # Get the watchers whichs use these, these should not be # active anymore for w in self.iter_watchers(): if 'circus.sockets.%s' % n.lower() in w.cmd: wn_with_deleted_socket.add(w.name) del self.sockets[s.name] # get added sockets for n in added_sn: socket_config = self.get_socket_config(new_cfg, n) s = CircusSocket.load_from_config(socket_config) s.bind_and_listen() self.sockets[s.name] = s if added_sn or deleted_sn: # make sure all existing watchers get the new sockets in # their attributes and get the old removed # XXX: is this necessary? self.sockets is an mutable # object for watcher in self.iter_watchers(): # XXX: What happens as initalize is called on a # running watcher? watcher.initialize(self.evpub_socket, self.sockets, self) # Gather watcher names. current_wn = set([i.name for i in self.iter_watchers()]) - ignore_wn new_wn = set([i['name'] for i in new_cfg.get('watchers', [])]) new_wn = new_wn | set([i['name'] for i in new_cfg.get('plugins', [])]) added_wn = (new_wn - current_wn) | wn_with_changed_socket deleted_wn = current_wn - new_wn - wn_with_changed_socket maybechanged_wn = current_wn - deleted_wn changed_wn = set([]) if wn_with_deleted_socket and wn_with_deleted_socket not in new_wn: raise ValueError('Watchers %s uses a socket which is deleted' % wn_with_deleted_socket) # get changed watchers for n in maybechanged_wn: w = self.get_watcher(n) new_watcher_cfg = (self.get_watcher_config(new_cfg, n) or self.get_plugin_config(new_cfg, n)) old_watcher_cfg = w._cfg.copy() if 'env' in new_watcher_cfg: new_watcher_cfg['env'] = parse_env_dict(new_watcher_cfg['env']) # discarding env exceptions for key in _ENV_EXCEPTIONS: if 'env' in new_watcher_cfg and key in new_watcher_cfg['env']: del new_watcher_cfg['env'][key] if 'env' in new_watcher_cfg and key in old_watcher_cfg['env']: del old_watcher_cfg['env'][key] diff = DictDiffer(new_watcher_cfg, old_watcher_cfg).changed() if diff == set(['numprocesses']): # if nothing but the number of processes is # changed, just changes this w.set_numprocesses(int(new_watcher_cfg['numprocesses'])) changed = False else: changed = len(diff) > 0 if changed: # Others things are changed. Just delete and add the watcher. changed_wn.add(n) deleted_wn.add(n) added_wn.add(n) # delete watchers for n in deleted_wn: w = self.get_watcher(n) yield w._stop() del self._watchers_names[w.name.lower()] self.watchers.remove(w) # add watchers for n in added_wn: new_watcher_cfg = (self.get_plugin_config(new_cfg, n) or self.get_watcher_config(new_cfg, n)) w = Watcher.load_from_config(new_watcher_cfg) w.initialize(self.evpub_socket, self.sockets, self) yield self.start_watcher(w) self.watchers.append(w) self._watchers_names[w.name.lower()] = w
def reload_from_config(self, config_file=None): cfg = get_config(config_file if config_file else self.config_file) # if arbiter is changed, reload everything if self.cfg2dict(cfg) != self.cfg: return True current_socket_names = set([i.name for i in self.sockets]) new_socket_names = set([i['name'] for i in cfg.get('sockets', [])]) added_socket_names = new_socket_names - current_socket_names deleted_socket_names = current_socket_names - new_socket_names maybechanged_socket_names = current_socket_names - deleted_socket_names changed_socket_names = set([]) watcher_names_with_changed_socket = set([]) watcher_names_with_deleted_socket = set([]) # get changed sockets for n in maybechanged_socket_names: s = self.get_socket(n) if s.cfg2dict(self.get_socket_config(cfg, n)) != s.cfg: changed_socket_names.add(n) # just delete the socket and add it again deleted_socket_names.add(n) added_socket_names.add(n) # Get the watchers whichs use these, so they could be deleted and added also for w in self.iter_watchers: if 'circus.sockets.%s' % n.lower() in w.cmd: watcher_names_with_changed_socket.add(w.name) # get deleted sockets for n in deleted_socket_names: s = self.get_socket(n) s.close() # Get the watchers whichs use these, these should not be active anymore for w in self.iter_watchers(): if 'circus.sockets.%s' % n.lower() in w.cmd: watcher_names_with_deleted_socket.add(w.name) self.sockets.remove(s) # get added sockets for n in added_socket_names: s = CircusSocket.load_from_config(self.get_socket_config(cfg, n)) s.bind_and_listen() self.sockets.append(s) if added_socket_names or deleted_socket_names: # make sure all existing watchers get the new sockets in their attributes and get the old removed for watcher in self.iter_watchers(): watcher.initialize(self.evpub_socket, self.sockets, self) current_watcher_names = set([i.name for i in self.iter_watchers()]) new_watcher_names = set([i['name'] for i in cfg.get('watchers', [])]) added_watcher_names = (new_watcher_names - current_watcher_names) | watcher_names_with_changed_socket deleted_watcher_names = current_watcher_names - new_watcher_names - watcher_names_with_changed_socket maybechanged_watcher_names = current_watcher_names - deleted_watcher_names changed_watcher_names = set([]) if watcher_names_with_deleted_socket and watcher_names_with_deleted_socket not in new_watcher_names: raise ValueError('Watchers %s uses a socket which is deleted' % watcher_names_with_deleted_socket) #get changed watchers for n in maybechanged_watcher_names: w = self.get_watcher(n) new_cfg = w.cfg2dict(self.get_watcher_config(cfg, n)) old_cfg = w.cfg2dict(w.cfg) # cfg2dict is used to make sure a copy is returned if new_cfg != old_cfg: old_cfg['numprocesses'] = new_cfg['numprocesses'] if new_cfg == old_cfg: # if nothing but the number of processes is changed, just changes this w.set_numprocesses(int(new_cfg['numprocesses'])) else: # Other thing are changed. Just delete and add the watcher. changed_watcher_names.add(n) deleted_watcher_names.add(n) added_watcher_names.add(n) # get deleted watchers for n in deleted_watcher_names: w = self.get_watcher(n) w.stop() del self._watchers_names[w.name.lower()] self.watchers.remove(w) # get added watchers for n in added_watcher_names: w = Watcher.load_from_config(self.get_watcher_config(cfg, n)) w.initialize(self.evpub_socket, self.sockets, self) w.start() self.watchers.append(w) self._watchers_names[w.name.lower()] = w return False
def test_load_from_config_unknown_proto(self): """Unknown proto in the config raises an error.""" config = {'name': '', 'proto': 'foo'} with self.assertRaises(socket.error): sock = CircusSocket.load_from_config(config)