def bind_and_listen(self):
try:
if self.is_unix:
if os.path.exists(self.path):
raise OSError("%r already exists. You might want to "
"remove it. If it's a stalled socket "
"file, just restart Circus" % self.path)
if self.umask is None:
self.bind(self.path)
else:
old_mask = os.umask(self.umask)
self.bind(self.path)
os.umask(old_mask)
else:
self.bind((self.host, self.port))
except socket.error:
logger.error('Could not bind %s' % self.location)
raise
self.setblocking(0)
if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
self.listen(self.backlog)
if not self.is_unix:
self.host, self.port = self.getsockname()
logger.debug('Socket bound at %s - fd: %d' % (self.location,
self.fileno()))
def bind_and_listen(self):
try:
if self.is_unix:
if os.path.exists(self.path):
raise OSError("%r already exists. You might want to "
"remove it. If it's a stalled socket "
"file, just restart Circus" % self.path)
if self.umask is None:
self.bind(self.path)
else:
old_mask = os.umask(self.umask)
self.bind(self.path)
os.umask(old_mask)
else:
self.bind((self.host, self.port))
except socket.error:
logger.error('Could not bind %s' % self.location)
raise
self.setblocking(0)
if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
self.listen(self.backlog)
if not self.is_unix:
self.host, self.port = self.getsockname()
logger.debug('Socket bound at %s - fd: %d' %
(self.location, self.fileno()))
def handle_recv(self, data):
"""Handle received message from circusd
We need to handle two messages:
- spawn: add a new monitored child pid
- reap: remove a killed child pid from monitoring
"""
topic, msg = data
topic_parts = topic.split(".")
logger.debug("received data from circusd: %s, %s", topic_parts, msg)
# check if monitored watchers:
if (topic_parts[0] == 'watcher' and
self._match_watcher_name(topic_parts[1])):
try:
message = json.loads(msg)
except ValueError:
logger.error("Error while decoding json for message: %s",
msg)
else:
if "process_pid" not in message:
logger.warning('no process_pid in message')
return
pid = str(message.get("process_pid"))
if topic_parts[2] == "spawn":
self.pid_status[pid] = dict(watcher=topic_parts[1],
last_activity=time.time())
logger.info("added new monitored pid for %s:%s",
topic_parts[1],
pid)
elif topic_parts[2] == "reap":
old_pid = self.pid_status.pop(pid)
logger.info("removed monitored pid for %s:%s",
old_pid['watcher'],
pid)
def collect_stats(self):
stats = self.call("stats", name=self.watcher)
if stats["status"] == "error":
self.statsd.increment("_resource_watcher.%s.error" % self.watcher)
logger.error("Can't get stats of %s" % self.watcher)
return
stats = dict((k, v) for k, v in six.iteritems(stats['info']) if
type(v) == dict)
# Convert absolute memory to bytes
for item in stats:
if stats[item]['mem_info1'] == 'N/A':
stats[item]['mem_abs'] = 'N/A'
else:
stats[item]['mem_abs'] = human2bytes(stats[item]['mem_info1'])
# Compute watcher stats if not in per_process mode
if not self.per_process:
stats[self.watcher] = defaultdict(lambda: 'N/A')
for item in filter(lambda x: x != self.watcher, stats):
for k in ['cpu', 'mem', 'mem_abs']:
if stats[item][k] != 'N/A':
if stats[self.watcher][k] == 'N/A':
stats[self.watcher][k] = stats[item][k]
else:
stats[self.watcher][k] += stats[item][k]
return stats
def dispatch(self, cid, msg):
try:
json_msg = json.loads(msg)
except ValueError:
raise gen.Return(self.send_error(cid, msg, "json invalid",
errno=errors.INVALID_JSON))
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
cast = json_msg.get('msg_type') == "cast"
resp = yield gen.Task(self.execute_command, cmd_name, properties, cid, msg, cast)
if resp is None:
resp = ok()
if not isinstance(resp, (dict, list,)):
msg = "msg %r tried to send a non-dict: %s" % (msg, str(resp))
logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
raise gen.Return(self.send_error(cid, msg, "server error", cast=cast,
errno=errors.BAD_MSG_DATA_ERROR))
if isinstance(resp, list):
resp = {"results": resp}
self.send_ok(cid, msg, resp, cast=cast)
def handle_recv(self, data):
"""Handle received message from circusd
We need to handle two messages:
- spawn: add a new monitored child pid
- reap: remove a killed child pid from monitoring
"""
topic, msg = data
topic_parts = topic.split(".")
logger.debug("received data from circusd: %s, %s", topic_parts, msg)
# check if monitored watchers:
if (topic_parts[0] == 'watcher'
and self._match_watcher_name(topic_parts[1])):
try:
message = json.loads(msg)
except ValueError:
logger.error("Error while decoding json for message: %s", msg)
else:
if "process_pid" not in message:
logger.warning('no process_pid in message')
return
pid = str(message.get("process_pid"))
if topic_parts[2] == "spawn":
self.pid_status[pid] = dict(watcher=topic_parts[1],
last_activity=time.time())
logger.info("added new monitored pid for %s:%s",
topic_parts[1], pid)
elif topic_parts[2] == "reap":
old_pid = self.pid_status.pop(pid)
logger.info("removed monitored pid for %s:%s",
old_pid['watcher'], pid)
def _dispatch_callback(self, msg, cid, mid, cast, cmd_name, resp=None):
if resp is None:
resp = ok()
if not isinstance(resp, (
dict,
list,
)):
msg = "msg %r tried to send a non-dict: %s" % (msg, str(resp))
logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
return self.send_error(mid,
cid,
msg,
"server error",
cast=cast,
errno=errors.BAD_MSG_DATA_ERROR)
if isinstance(resp, list):
resp = {"results": resp}
self.send_ok(mid, cid, msg, resp, cast=cast)
if cmd_name.lower() == "quit":
if cid is not None:
self.stream.flush()
self.arbiter.stop()
def handle_recv(self, data):
"""Handle received message from circusd
We need to handle two messages:
- spawn: add a new monitored child pid
- reap: remove a killed child pid from monitoring
"""
watcher_name, action, msg = self.split_data(data)
logger.debug("received data from circusd: watcher.%s.%s, %s",
watcher_name, action, msg)
# check if monitored watchers:
if self._match_watcher_name(watcher_name):
try:
message = self.load_message(msg)
except ValueError:
logger.error("Error while decoding json for message: %s", msg)
else:
if "process_pid" not in message:
logger.warning('no process_pid in message')
return
pid = str(message.get("process_pid"))
if action == "spawn":
self.pid_status[pid] = dict(watcher=watcher_name,
last_activity=time.time())
logger.info("added new monitored pid for %s:%s",
watcher_name, pid)
# very questionable fix for Py3 here!
# had to add check for pid in self.pid_status
elif action == "reap" and pid in self.pid_status:
old_pid = self.pid_status.pop(pid)
logger.info("removed monitored pid for %s:%s",
old_pid['watcher'], pid)
def bind_and_listen(self):
try:
if self.is_unix:
if os.path.exists(self.path):
raise OSError("%r already exists. You might want to "
"remove it. If it's a stalled socket "
"file, just restart Circus" % self.path)
if self.umask is None:
self.bind(self.path)
else:
old_mask = os.umask(self.umask)
self.bind(self.path)
os.umask(old_mask)
else:
if self.interface is not None:
# Bind to device if given, e.g. to limit which device to bind
# when binding on IN_ADDR_ANY or IN_ADDR_BROADCAST.
import IN
self.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE,
self.interface + '\0')
logger.debug('Binding to device: %s' % self.interface)
self.bind((self.host, self.port))
except socket.error:
logger.error('Could not bind %s' % self.location)
raise
self.setblocking(0)
if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
self.listen(self.backlog)
if not self.is_unix:
self.host, self.port = self.getsockname()
logger.debug('Socket bound at %s - fd: %d' % (self.location,
self.fileno()))
def handle_recv(self, data):
"""Handle received message from circusd
We need to handle two messages:
- spawn: add a new monitored child pid
- reap: remove a killed child pid from monitoring
"""
watcher_name, action, msg = self.split_data(data)
logger.debug("received data from circusd: watcher.%s.%s, %s",
watcher_name, action, msg)
# check if monitored watchers:
if self._match_watcher_name(watcher_name):
try:
message = self.load_message(msg)
except ValueError:
logger.error("Error while decoding json for message: %s",
msg)
else:
if "process_pid" not in message:
logger.warning('no process_pid in message')
return
pid = str(message.get("process_pid"))
if action == "spawn":
self.pid_status[pid] = dict(watcher=watcher_name,
last_activity=time.time())
logger.info("added new monitored pid for %s:%s",
watcher_name,
pid)
# very questionable fix for Py3 here!
# had to add check for pid in self.pid_status
elif action == "reap" and pid in self.pid_status:
old_pid = self.pid_status.pop(pid)
logger.info("removed monitored pid for %s:%s",
old_pid['watcher'],
pid)
def bind_and_listen(self):
try:
if self.is_unix:
if os.path.exists(self.path):
raise OSError("%r already exists. You might want to "
"remove it. If it's a stalled socket "
"file, just restart Circus" % self.path)
if self.umask is None:
self.bind(self.path)
else:
old_mask = os.umask(self.umask)
self.bind(self.path)
os.umask(old_mask)
else:
if self.interface is not None:
# Bind to device if given, e.g. to limit which device to bind
# when binding on IN_ADDR_ANY or IN_ADDR_BROADCAST.
import IN
self.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE,
self.interface + '\0')
logger.debug('Binding to device: %s' % self.interface)
self.bind((self.host, self.port))
except socket.error:
logger.error('Could not bind %s' % self.location)
raise
self.setblocking(0)
if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
self.listen(self.backlog)
if not self.is_unix:
self.host, self.port = self.getsockname()
logger.debug('Socket bound at %s - fd: %d' %
(self.location, self.fileno()))
def send_response(self, client_id, sock, msg, resp):
try:
sock.send(client_id, zmq.SNDMORE)
sock.send(resp)
except zmq.ZMQError as e:
logger.error("Received %r - Could not send back %r - %s" %
(msg, resp, str(e)))
def dispatch(self, job):
cid, msg = job
try:
json_msg = json.loads(msg)
except ValueError:
return self.send_error(cid, msg, "json invalid",
errno=errors.INVALID_JSON)
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
cast = json_msg.get('msg_type') == "cast"
try:
if cmd_name is None:
error = "no cmd: %r" % cmd_name
return self.send_error(cid, msg, error, cast=cast,
errno=errors.UNKNOWN_COMMAND)
cmd = self.commands[cmd_name.lower()]
except KeyError:
error = "unknown command: %r" % cmd_name
return self.send_error(cid, msg, error, cast=cast,
errno=errors.UNKNOWN_COMMAND)
try:
cmd.validate(properties)
resp = cmd.execute(self.arbiter, properties)
except MessageError as e:
return self.send_error(cid, msg, str(e), cast=cast,
errno=errors.MESSAGE_ERROR)
except OSError as e:
return self.send_error(cid, msg, str(e), cast=cast,
errno=errors.OS_ERROR)
except:
exctype, value = sys.exc_info()[:2]
tb = traceback.format_exc()
reason = "command %r: %s" % (msg, value)
logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
return self.send_error(cid, msg, reason, tb, cast=cast,
errno=errors.COMMAND_ERROR)
if resp is None:
resp = ok()
if not isinstance(resp, (dict, list,)):
msg = "msg %r tried to send a non-dict: %s" % (msg, str(resp))
logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
return self.send_error(cid, msg, "server error", cast=cast,
errno=errors.BAD_MSG_DATA_ERROR)
if isinstance(resp, list):
resp = {"results": resp}
self.send_ok(cid, msg, resp, cast=cast)
if cmd_name.lower() == "quit":
if cid is not None:
self.stream.flush()
self.arbiter.stop()
def load_from_config(cls, config_file):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
ssh_server=cfg.get('ssh_server', None))
return arbiter
def load_from_config(cls, config_file):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
stats_endpoint=cfg.get('stats_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
ssh_server=cfg.get('ssh_server', None))
return arbiter
def dispatch(self, job):
cid, msg = job
try:
json_msg = json.loads(msg)
except ValueError:
return self.send_error(cid, msg, "json invalid",
errno=errors.INVALID_JSON)
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
cast = json_msg.get('msg_type') == "cast"
try:
cmd = self.commands[cmd_name.lower()]
except KeyError:
error = "unknown command: %r" % cmd_name
return self.send_error(cid, msg, error, cast=cast,
errno=errors.UNKNOWN_COMMAND)
try:
cmd.validate(properties)
resp = cmd.execute(self.arbiter, properties)
except MessageError as e:
return self.send_error(cid, msg, str(e), cast=cast,
errno=errors.MESSAGE_ERROR)
except OSError as e:
return self.send_error(cid, msg, str(e), cast=cast,
errno=errors.OS_ERROR)
except:
exctype, value = sys.exc_info()[:2]
tb = traceback.format_exc()
reason = "command %r: %s" % (msg, value)
logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
return self.send_error(cid, msg, reason, tb, cast=cast,
errno=errors.COMMAND_ERROR)
if resp is None:
resp = ok()
if not isinstance(resp, (dict, list,)):
msg = "msg %r tried to send a non-dict: %s" % (msg,
str(resp))
logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
return self.send_error(cid, msg, "server error", cast=cast,
errno=errors.BAD_MSG_DATA_ERROR)
if isinstance(resp, list):
resp = {"results": resp}
self.send_ok(cid, msg, resp, cast=cast)
if cmd_name.lower() == "quit":
if cid is not None:
self.stream.flush()
self.arbiter.stop()
def check_future_exception_and_log(future):
if isinstance(future, concurrent.Future):
exception = future.exception()
if exception is not None:
logger.error("exception %s caught" % exception)
if hasattr(future, "exc_info"):
exc_info = future.exc_info()
traceback.print_tb(exc_info[2])
return exception
def check_future_exception_and_log(future):
if isinstance(future, concurrent.Future):
exception = future.exception()
if exception is not None:
logger.error("exception %s caught" % exception)
if hasattr(future, "exc_info"):
exc_info = future.exc_info()
traceback.print_tb(exc_info[2])
return exception
def children_started(watcher, arbiter, hook_name, pid, **kwargs):
name = watcher.name
wid = watcher.processes[pid].wid
try:
wait_for_workers(name, wid, 'running', timeout_seconds=10,
minimum_age=5)
return True
except TimeoutError:
logger.error('%s children are flapping on %d', name, pid)
return False
def get_worker_states(name, wid, minimum_age=0.0):
stats = get_uwsgi_stats(name, wid)
if 'workers' not in stats:
logger.error("Error: No workers found for WID %d of %d", wid, name)
return ['unknown']
workers = stats['workers']
return [
worker["status"] if 'status' in worker and worker['last_spawn'] < time() - minimum_age else 'unknown'
for worker in workers
]
def get_worker_states(name, wid, base_port, minimum_age=0.0):
stats = get_uwsgi_stats(name, wid, base_port)
if 'workers' not in stats:
logger.error("Error: No workers found for WID %d of %d", wid, name)
return ['unknown']
workers = stats['workers']
return [
worker["status"] if 'status' in worker
and worker['last_spawn'] < time() - minimum_age else 'unknown'
for worker in workers
]
def load_from_config(cls, config_file, loop=None):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket_ in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket_))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers,
cfg['endpoint'],
cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
statsd=cfg.get('statsd', False),
stats_endpoint=cfg.get('stats_endpoint'),
papa_endpoint=cfg.get('papa_endpoint'),
multicast_endpoint=cfg.get('multicast_endpoint'),
plugins=cfg.get('plugins'),
sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
loop=loop,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
debug_gc=cfg.get('debug_gc', False),
ssh_server=cfg.get('ssh_server', None),
pidfile=cfg.get('pidfile', None),
loglevel=cfg.get('loglevel', None),
logoutput=cfg.get('logoutput', None),
loggerconfig=cfg.get('loggerconfig', None),
fqdn_prefix=cfg.get('fqdn_prefix', None),
umask=cfg['umask'],
endpoint_owner=cfg.get('endpoint_owner', None))
# store the cfg which will be used, so it can be used later
# for checking if the cfg has been changed
arbiter._cfg = cls.get_arbiter_config(cfg)
arbiter.config_file = config_file
return arbiter
def bind_and_listen(self):
try:
self.bind((self.host, self.port))
except socket.error:
logger.error('Could not bind %s:%d' % (self.host, self.port))
raise
self.setblocking(0)
self.listen(self.backlog)
self.host, self.port = self.getsockname()
logger.debug('Socket bound at %s:%d - fd: %d' % (self.host, self.port,
self.fileno()))
def signal(self, sig, frame):
signame = self.SIG_NAMES.get(sig)
if signame is not None:
try:
handler = getattr(self, "handle_%s" % signame)
handler()
except AttributeError:
pass
except Exception as e:
tb = traceback.format_exc()
logger.error("error: %s [%s]" % (e, tb))
sys.exit(1)
def signal(self, sig, frame):
signame = self.SIG_NAMES.get(sig)
if signame is not None:
try:
handler = getattr(self, "handle_%s" % signame)
handler()
except AttributeError:
pass
except Exception, e:
tb = traceback.format_exc()
logger.error("error: %s [%s]" % (e, tb))
sys.exit(1)
def reap_process(self, pid, status=None):
"""ensure that the process is killed (and not a zombie)"""
if pid not in self.processes:
return
process = self.processes.pop(pid)
if status is None:
while True:
try:
_, status = os.waitpid(pid, os.WNOHANG)
except OSError as e:
if e.errno == errno.EAGAIN:
time.sleep(0.001)
continue
elif e.errno == errno.ECHILD:
# nothing to do here, we do not have any child
# process running
# but we still need to send the "reap" signal.
#
# This can happen if poll() or wait() were called on
# the underlying process.
logger.error('reaping already dead process %s [%s]',
pid, self.name)
self.notify_event(
"reap",
{"process_pid": pid,
"time": time.time(),
"exit_code": process.returncode()})
process.stop()
return
else:
raise
# get return code
if os.WIFSIGNALED(status):
exit_code = os.WTERMSIG(status)
# process exited using exit(2) system call; return the
# integer exit(2) system call has been called with
elif os.WIFEXITED(status):
exit_code = os.WEXITSTATUS(status)
else:
# should never happen
raise RuntimeError("Unknown process exit status")
# if the process is dead or a zombie try to definitely stop it.
if process.status in (DEAD_OR_ZOMBIE, UNEXISTING):
process.stop()
logger.debug('reaping process %s [%s]', pid, self.name)
self.notify_event("reap",
{"process_pid": pid,
"time": time.time(),
"exit_code": exit_code})
def load_from_config(cls, config_file, loop=None):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get("watchers", []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket in cfg.get("sockets", []):
sockets.append(CircusSocket.load_from_config(socket))
httpd = cfg.get("httpd", False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error("You need to install circus-web")
sys.exit(1)
# creating arbiter
arbiter = cls(
watchers,
cfg["endpoint"],
cfg["pubsub_endpoint"],
check_delay=cfg.get("check_delay", 1.0),
prereload_fn=cfg.get("prereload_fn"),
statsd=cfg.get("statsd", False),
stats_endpoint=cfg.get("stats_endpoint"),
multicast_endpoint=cfg.get("multicast_endpoint"),
plugins=cfg.get("plugins"),
sockets=sockets,
warmup_delay=cfg.get("warmup_delay", 0),
httpd=httpd,
loop=loop,
httpd_host=cfg.get("httpd_host", "localhost"),
httpd_port=cfg.get("httpd_port", 8080),
debug=cfg.get("debug", False),
ssh_server=cfg.get("ssh_server", None),
pidfile=cfg.get("pidfile", None),
loglevel=cfg.get("loglevel", None),
logoutput=cfg.get("logoutput", None),
fqdn_prefix=cfg.get("fqdn_prefix", None),
umask=cfg["umask"],
)
# store the cfg which will be used, so it can be used later
# for checking if the cfg has been changed
arbiter._cfg = cls.get_arbiter_config(cfg)
arbiter.config_file = config_file
return arbiter
def children_started(watcher, arbiter, hook_name, pid, **kwargs):
name = watcher.name
wid = watcher.processes[pid].wid
try:
wait_for_workers(name,
wid,
'running',
timeout_seconds=10,
minimum_age=5)
return True
except TimeoutError:
logger.error('%s children are flapping on %d', name, pid)
return False
def bind_and_listen(self):
try:
self.bind((self.host, self.port))
except socket.error:
logger.error('Could not bind %s:%d' % (self.host, self.port))
raise
self.setblocking(0)
if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
self.listen(self.backlog)
self.host, self.port = self.getsockname()
logger.debug('Socket bound at %s:%d - fd: %d' %
(self.host, self.port, self.fileno()))
def load_from_config(cls, config_file, loop=None):
cfg = get_config(config_file)
watchers = []
for watcher in cfg.get('watchers', []):
watchers.append(Watcher.load_from_config(watcher))
sockets = []
for socket_ in cfg.get('sockets', []):
sockets.append(CircusSocket.load_from_config(socket_))
httpd = cfg.get('httpd', False)
if httpd:
# controlling that we have what it takes to run the web UI
# if something is missing this will tell the user
try:
import circusweb # NOQA
except ImportError:
logger.error('You need to install circus-web')
sys.exit(1)
# creating arbiter
arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
check_delay=cfg.get('check_delay', 1.),
prereload_fn=cfg.get('prereload_fn'),
statsd=cfg.get('statsd', False),
stats_endpoint=cfg.get('stats_endpoint'),
papa_endpoint=cfg.get('papa_endpoint'),
multicast_endpoint=cfg.get('multicast_endpoint'),
plugins=cfg.get('plugins'), sockets=sockets,
warmup_delay=cfg.get('warmup_delay', 0),
httpd=httpd,
loop=loop,
httpd_host=cfg.get('httpd_host', 'localhost'),
httpd_port=cfg.get('httpd_port', 8080),
debug=cfg.get('debug', False),
debug_gc=cfg.get('debug_gc', False),
ssh_server=cfg.get('ssh_server', None),
pidfile=cfg.get('pidfile', None),
loglevel=cfg.get('loglevel', None),
logoutput=cfg.get('logoutput', None),
loggerconfig=cfg.get('loggerconfig', None),
fqdn_prefix=cfg.get('fqdn_prefix', None),
umask=cfg['umask'],
endpoint_owner=cfg.get('endpoint_owner', None))
# store the cfg which will be used, so it can be used later
# for checking if the cfg has been changed
arbiter._cfg = cls.get_arbiter_config(cfg)
arbiter.config_file = config_file
return arbiter
def dispatch(self, job):
cid, msg = job
try:
json_msg = json.loads(msg)
except ValueError:
return self.send_error(cid, msg, "json invalid")
cmd_name = json_msg.get('command')
properties = json_msg.get('properties', {})
try:
cmd = self.commands[cmd_name.lower()]
except KeyError:
error = "unknown command: %r" % cmd_name
return self.send_error(cid, msg, error)
try:
cmd.validate(properties)
resp = cmd.execute(self.arbiter, properties)
except MessageError as e:
return self.send_error(cid, msg, str(e))
except OSError as e:
return self.send_error(cid, msg, str(e))
except:
exctype, value = sys.exc_info()[:2]
tb = traceback.format_exc()
reason = "command %r: %s" % (msg, value)
logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
return self.send_error(cid, msg, reason, tb)
if resp is None:
resp = ok()
if not isinstance(resp, (dict, list,)):
msg = "msg %r tried to send a non-dict: %s" % (msg,
str(resp))
logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
return self.send_error(cid, msg, "server error")
if isinstance(resp, list):
resp = {"results": resp}
self.send_ok(cid, msg, resp)
if cmd_name.lower() == "quit":
if cid is not None:
self.stream.flush()
self.arbiter.stop()
def get_uwsgi_stats(name, wid, base_port):
sock = socket.create_connection(('127.0.0.1', base_port + wid), timeout=1)
received = sock.recv(100000)
data = bytes()
while received:
data += received
received = sock.recv(100000)
if not data:
logger.error("Error: No stats seem available for WID %d of %s", wid,
name)
return
# recent versions of uWSGI had some garbage in the JSON so strip it out
data = data.decode('latin', 'replace')
data = NON_JSON_CHARACTERS.sub('', data)
return loads(data)
def children_started(watcher, arbiter, hook_name, pid, **kwargs):
name = watcher.name
wid = watcher.processes[pid].wid
base_port = int(watcher._options.get('stats_base_port', 8090))
logger.info('%s waiting for workers', name)
try:
wait_for_workers(name, wid, base_port, 'running', timeout_seconds=15,
minimum_age=5)
return True
except TimeoutError:
logger.error('%s children are flapping on %d', name, pid)
return False
except Exception:
logger.error('%s not publishing stats on %d', name, pid)
return False
def get_uwsgi_stats(name, wid, base_port):
sock = socket.create_connection(('127.0.0.1', base_port + wid), timeout=1)
received = sock.recv(100000)
data = bytes()
while received:
data += received
received = sock.recv(100000)
if not data:
logger.error(
"Error: No stats seem available for WID %d of %s", wid, name)
return
# recent versions of uWSGI had some garbage in the JSON so strip it out
if not PY2:
data = data.decode('latin', 'replace')
data = NON_JSON_CHARACTERS.sub('', data)
return loads(data)
def bind_and_listen(self):
try:
if self.is_unix:
if os.path.exists(self.path):
if self.replace:
os.unlink(self.path)
else:
raise OSError(
"%r already exists. You might want to "
"remove it. If it's a stalled socket "
"file, just restart Circus" % self.path
)
if self.umask is None:
self.bind(self.path)
else:
old_mask = os.umask(self.umask)
self.bind(self.path)
os.umask(old_mask)
else:
if self.interface is not None:
# Bind to device if given, e.g. to limit which device to
# bind when binding on IN_ADDR_ANY or IN_ADDR_BROADCAST.
import IN
if hasattr(IN, "SO_BINDTODEVICE"):
self.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE, self.interface + "\0")
logger.debug("Binding to device: %s" % self.interface)
self.bind((self.host, self.port))
except socket.error:
logger.error("Could not bind %s" % self.location)
raise
if self.blocking:
self.setblocking(1)
else:
self.setblocking(0)
if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
self.listen(self.backlog)
if not self.is_unix:
if self.family == socket.AF_INET6:
self.host, self.port, _flowinfo, _scopeid = self.getsockname()
else:
self.host, self.port = self.getsockname()
logger.debug("Socket bound at %s - fd: %d" % (self.location, self.fileno()))
def _setup_vhost(self, sockets, worker):
options = self.call('options', name=worker)
if not options or options.get('status') != 'ok':
logger.error('Failed to retrieve options')
return
options = options['options']
vhost = options.get('vhostino.vhost')
#logger.info('\n\nCONF %s -> %s', worker, options)
if vhost == 'True':
socket = sockets.get(worker)
if socket:
logger.info('Vhostino: Registering %s -> %s', worker, socket['port'])
self.server.config.add_vhost(worker, socket['port'])
if options.get('vhostino.default_vhost'):
logger.info('Vhostino: Default Vhost %s -> %s', worker, socket['port'])
self.server.config.set_default(socket['port'])
def _initialize_proxy(self):
sockets = self.call('listsockets')
if not sockets or sockets.get('status') != 'ok':
logger.error('Failed to retrieve existing sockets list')
return
sockets = self._sockets_by_name(sockets['sockets'])
status = self.call('status')
if not status and status.get('status') != 'ok':
logger.error('Failed to retrieve status')
return
for worker in status['statuses']:
self._setup_vhost(sockets, worker)
logger.info('VHosting: Serving on %d', self.server.server_port)
self.server.serve_forever()
def _dispatch_callback(self, msg, cid, mid, cast, cmd_name, resp=None):
if resp is None:
resp = ok()
if not isinstance(resp, (dict, list)):
msg = "msg %r tried to send a non-dict: %s" % (msg, str(resp))
logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
return self.send_error(mid, cid, msg, "server error", cast=cast,
errno=errors.BAD_MSG_DATA_ERROR)
if isinstance(resp, list):
resp = {"results": resp}
self.send_ok(mid, cid, msg, resp, cast=cast)
if cmd_name.lower() == "quit":
if cid is not None:
self.stream.flush()
def clean_stop(watcher, arbiter, hook_name, pid, signum, **kwargs):
if len(watcher.processes) > watcher.numprocesses and signum == signal.SIGQUIT:
name = watcher.name
started = watcher.processes[pid].started
newer_pids = [p for p, w in watcher.processes.items() if p != pid and w.started > started]
# if the one being stopped is actually the newer one, just do it
if len(newer_pids) < watcher.numprocesses:
return True
wid = watcher.processes[pid].wid
base_port = int(watcher._options.get('stats_base_port', 8090))
logger.info('%s pausing', name)
try:
watcher.send_signal(pid, signal.SIGTSTP)
wait_for_workers(name, wid, base_port, 'paused')
logger.info('%s workers idle', name)
except Exception as e:
logger.error('trouble pausing %s: %s', name, e)
return True
def _bind_socket(self):
"""bind the listening socket for watchdog udp and start an event
handler for handling udp received messages.
"""
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
self.sock.bind((self.watchdog_ip, self.watchdog_port))
except socket.error as socket_error:
logger.error(
"Problem while binding watchdog socket on %s:%s (err %s",
self.watchdog_ip, self.watchdog_port, str(socket_error))
self.sock = None
else:
self.sock.settimeout(1)
self.loop.add_handler(self.sock.fileno(), self.receive_udp_socket,
ioloop.IOLoop.READ)
logger.info("Watchdog listening UDP on %s:%s", self.watchdog_ip,
self.watchdog_port)
def bind_and_listen(self):
try:
if self.is_unix:
self.bind(self.path)
else:
self.bind((self.host, self.port))
except socket.error:
logger.error('Could not bind %s' % self.location)
raise
self.setblocking(0)
if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
self.listen(self.backlog)
if not self.is_unix:
self.host, self.port = self.getsockname()
logger.debug('Socket bound at %s - fd: %d' % (self.location,
self.fileno()))
def _bind_socket(self):
"""bind the listening socket for watchdog udp and start an event
handler for handling udp received messages.
"""
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
self.sock.bind((self.watchdog_ip, self.watchdog_port))
except socket.error as socket_error:
logger.error(
"Problem while binding watchdog socket on %s:%s (err %s",
self.watchdog_ip,
self.watchdog_port,
str(socket_error),
)
self.sock = None
else:
self.sock.settimeout(1)
self.loop.add_handler(self.sock.fileno(), self.receive_udp_socket, ioloop.IOLoop.READ)
logger.info("Watchdog listening UDP on %s:%s", self.watchdog_ip, self.watchdog_port)
def children_started(watcher, arbiter, hook_name, pid, **kwargs):
name = watcher.name
wid = watcher.processes[pid].wid
base_port = int(watcher._options.get('stats_base_port', 8090))
logger.info('%s waiting for workers', name)
try:
wait_for_workers(name,
wid,
base_port,
'running',
timeout_seconds=15,
minimum_age=5)
return True
except TimeoutError:
logger.error('%s children are flapping on %d', name, pid)
return False
except Exception:
logger.error('%s not publishing stats on %d', name, pid)
return False
def get_uwsgi_stats(name, wid):
try:
sock = socket.create_connection(('127.0.0.1', 8090 + wid), timeout=1)
except Exception as e:
logger.error(
"Error: Connection refused for {0}} on 127.0.0.1:809{1} - {2}".
format(name, wid, e))
raise e
received = sock.recv(100000)
data = bytes()
while received:
data += received
received = sock.recv(100000)
if not data:
logger.error("Error: No stats seem available for WID %d of %s", wid,
name)
return
# recent versions of uWSGI had some garbage in the JSON so strip it out
data = NON_JSON_CHARACTERS.sub('', data.decode())
return loads(data)
def get_uwsgi_stats(name, wid):
try:
sock = socket.create_connection(('127.0.0.1', 8090 + wid),
timeout=1)
except Exception as e:
logger.error(
"Error: Connection refused for {0}} on 127.0.0.1:809{1} - {2}"
.format(name, wid, e))
raise e
received = sock.recv(100000)
data = bytes()
while received:
data += received
received = sock.recv(100000)
if not data:
logger.error(
"Error: No stats seem available for WID %d of %s", wid, name)
return
# recent versions of uWSGI had some garbage in the JSON so strip it out
data = NON_JSON_CHARACTERS.sub('', data.decode())
return loads(data)
def clean_stop(watcher, arbiter, hook_name, pid, signum, **kwargs):
if len(watcher.processes
) > watcher.numprocesses and signum == signal.SIGQUIT:
name = watcher.name
started = watcher.processes[pid].started
newer_pids = [
p for p, w in watcher.processes.items()
if p != pid and w.started > started
]
# if the one being stopped is actually the newer one, just do it
if len(newer_pids) < watcher.numprocesses:
return True
wid = watcher.processes[pid].wid
base_port = int(watcher._options.get('stats_base_port', 8090))
logger.info('%s pausing', name)
try:
watcher.send_signal(pid, signal.SIGTSTP)
wait_for_workers(name, wid, base_port, 'paused')
logger.info('%s workers idle', name)
except Exception as e:
logger.error('trouble pausing %s: %s', name, e)
return True
def handle_callback_exception(callback):
logger.error("Exception in callback %r", callback, exc_info=True)
