예제 #1
0
    def bind_and_listen(self):
        try:
            if self.is_unix:
                if os.path.exists(self.path):
                    raise OSError("%r already exists. You might want to "
                                  "remove it. If it's a stalled socket "
                                  "file, just restart Circus" % self.path)
                if self.umask is None:
                    self.bind(self.path)
                else:
                    old_mask = os.umask(self.umask)
                    self.bind(self.path)
                    os.umask(old_mask)
            else:
                self.bind((self.host, self.port))
        except socket.error:
            logger.error('Could not bind %s' % self.location)
            raise

        self.setblocking(0)
        if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
            self.listen(self.backlog)

        if not self.is_unix:
            self.host, self.port = self.getsockname()

        logger.debug('Socket bound at %s - fd: %d' % (self.location,
                                                      self.fileno()))
예제 #2
0
    def bind_and_listen(self):
        try:
            if self.is_unix:
                if os.path.exists(self.path):
                    raise OSError("%r already exists. You might want to "
                                  "remove it. If it's a stalled socket "
                                  "file, just restart Circus" % self.path)
                if self.umask is None:
                    self.bind(self.path)
                else:
                    old_mask = os.umask(self.umask)
                    self.bind(self.path)
                    os.umask(old_mask)
            else:
                self.bind((self.host, self.port))
        except socket.error:
            logger.error('Could not bind %s' % self.location)
            raise

        self.setblocking(0)
        if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
            self.listen(self.backlog)

        if not self.is_unix:
            self.host, self.port = self.getsockname()

        logger.debug('Socket bound at %s - fd: %d' %
                     (self.location, self.fileno()))
예제 #3
0
파일: watchdog.py 프로젝트: B-Rich/circus
    def handle_recv(self, data):
        """Handle received message from circusd

        We need to handle two messages:
        - spawn: add a new monitored child pid
        - reap: remove a killed child pid from monitoring
        """
        topic, msg = data
        topic_parts = topic.split(".")
        logger.debug("received data from circusd: %s, %s", topic_parts, msg)
        # check if monitored watchers:
        if (topic_parts[0] == 'watcher' and
                self._match_watcher_name(topic_parts[1])):
            try:
                message = json.loads(msg)
            except ValueError:
                logger.error("Error while decoding json for message: %s",
                             msg)
            else:
                if "process_pid" not in message:
                    logger.warning('no process_pid in message')
                    return
                pid = str(message.get("process_pid"))
                if topic_parts[2] == "spawn":
                    self.pid_status[pid] = dict(watcher=topic_parts[1],
                                                last_activity=time.time())
                    logger.info("added new monitored pid for %s:%s",
                                topic_parts[1],
                                pid)
                elif topic_parts[2] == "reap":
                    old_pid = self.pid_status.pop(pid)
                    logger.info("removed monitored pid for %s:%s",
                                old_pid['watcher'],
                                pid)
예제 #4
0
    def collect_stats(self):
        stats = self.call("stats", name=self.watcher)
        if stats["status"] == "error":
            self.statsd.increment("_resource_watcher.%s.error" % self.watcher)
            logger.error("Can't get stats of %s" % self.watcher)
            return
        stats = dict((k, v) for k, v in six.iteritems(stats['info']) if
                     type(v) == dict)

        # Convert absolute memory to bytes
        for item in stats:
            if stats[item]['mem_info1'] == 'N/A':
                stats[item]['mem_abs'] = 'N/A'
            else:
                stats[item]['mem_abs'] = human2bytes(stats[item]['mem_info1'])

        # Compute watcher stats if not in per_process mode
        if not self.per_process:
            stats[self.watcher] = defaultdict(lambda: 'N/A')
            for item in filter(lambda x: x != self.watcher, stats):
                for k in ['cpu', 'mem', 'mem_abs']:
                    if stats[item][k] != 'N/A':
                        if stats[self.watcher][k] == 'N/A':
                            stats[self.watcher][k] = stats[item][k]
                        else:
                            stats[self.watcher][k] += stats[item][k]
        return stats
예제 #5
0
    def dispatch(self, cid, msg):
        try:
            json_msg = json.loads(msg)
        except ValueError:
            raise gen.Return(self.send_error(cid, msg, "json invalid",
                                             errno=errors.INVALID_JSON))

        cmd_name = json_msg.get('command')
        properties = json_msg.get('properties', {})
        cast = json_msg.get('msg_type') == "cast"

        resp = yield gen.Task(self.execute_command, cmd_name, properties, cid, msg, cast)

        if resp is None:
            resp = ok()

        if not isinstance(resp, (dict, list,)):
            msg = "msg %r tried to send a non-dict: %s" % (msg, str(resp))
            logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
            raise gen.Return(self.send_error(cid, msg, "server error", cast=cast,
                                             errno=errors.BAD_MSG_DATA_ERROR))

        if isinstance(resp, list):
            resp = {"results": resp}

        self.send_ok(cid, msg, resp, cast=cast)
예제 #6
0
파일: watchdog.py 프로젝트: suligap/circus
    def handle_recv(self, data):
        """Handle received message from circusd

        We need to handle two messages:
        - spawn: add a new monitored child pid
        - reap: remove a killed child pid from monitoring
        """
        topic, msg = data
        topic_parts = topic.split(".")
        logger.debug("received data from circusd: %s, %s", topic_parts, msg)
        # check if monitored watchers:
        if (topic_parts[0] == 'watcher'
                and self._match_watcher_name(topic_parts[1])):
            try:
                message = json.loads(msg)
            except ValueError:
                logger.error("Error while decoding json for message: %s", msg)
            else:
                if "process_pid" not in message:
                    logger.warning('no process_pid in message')
                    return
                pid = str(message.get("process_pid"))
                if topic_parts[2] == "spawn":
                    self.pid_status[pid] = dict(watcher=topic_parts[1],
                                                last_activity=time.time())
                    logger.info("added new monitored pid for %s:%s",
                                topic_parts[1], pid)
                elif topic_parts[2] == "reap":
                    old_pid = self.pid_status.pop(pid)
                    logger.info("removed monitored pid for %s:%s",
                                old_pid['watcher'], pid)
예제 #7
0
    def _dispatch_callback(self, msg, cid, mid, cast, cmd_name, resp=None):
        if resp is None:
            resp = ok()

        if not isinstance(resp, (
                dict,
                list,
        )):
            msg = "msg %r tried to send a non-dict: %s" % (msg, str(resp))
            logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
            return self.send_error(mid,
                                   cid,
                                   msg,
                                   "server error",
                                   cast=cast,
                                   errno=errors.BAD_MSG_DATA_ERROR)

        if isinstance(resp, list):
            resp = {"results": resp}

        self.send_ok(mid, cid, msg, resp, cast=cast)

        if cmd_name.lower() == "quit":
            if cid is not None:
                self.stream.flush()

            self.arbiter.stop()
예제 #8
0
    def handle_recv(self, data):
        """Handle received message from circusd

        We need to handle two messages:
        - spawn: add a new monitored child pid
        - reap: remove a killed child pid from monitoring
        """
        watcher_name, action, msg = self.split_data(data)
        logger.debug("received data from circusd: watcher.%s.%s, %s",
                     watcher_name, action, msg)
        # check if monitored watchers:
        if self._match_watcher_name(watcher_name):
            try:
                message = self.load_message(msg)
            except ValueError:
                logger.error("Error while decoding json for message: %s", msg)
            else:
                if "process_pid" not in message:
                    logger.warning('no process_pid in message')
                    return
                pid = str(message.get("process_pid"))
                if action == "spawn":
                    self.pid_status[pid] = dict(watcher=watcher_name,
                                                last_activity=time.time())
                    logger.info("added new monitored pid for %s:%s",
                                watcher_name, pid)
                # very questionable fix for Py3 here!
                # had to add check for pid in self.pid_status
                elif action == "reap" and pid in self.pid_status:
                    old_pid = self.pid_status.pop(pid)
                    logger.info("removed monitored pid for %s:%s",
                                old_pid['watcher'], pid)
예제 #9
0
    def bind_and_listen(self):
        try:
            if self.is_unix:
                if os.path.exists(self.path):
                    raise OSError("%r already exists. You might want to "
                                  "remove it. If it's a stalled socket "
                                  "file, just restart Circus" % self.path)
                if self.umask is None:
                    self.bind(self.path)
                else:
                    old_mask = os.umask(self.umask)
                    self.bind(self.path)
                    os.umask(old_mask)
            else:
                if self.interface is not None:
                    # Bind to device if given, e.g. to limit which device to bind
                    # when binding on IN_ADDR_ANY or IN_ADDR_BROADCAST.
                    import IN
                    self.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE,
                        self.interface + '\0')
                    logger.debug('Binding to device: %s' % self.interface)
                self.bind((self.host, self.port))
        except socket.error:
            logger.error('Could not bind %s' % self.location)
            raise

        self.setblocking(0)
        if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
            self.listen(self.backlog)

        if not self.is_unix:
            self.host, self.port = self.getsockname()

        logger.debug('Socket bound at %s - fd: %d' % (self.location,
                                                      self.fileno()))
예제 #10
0
    def handle_recv(self, data):
        """Handle received message from circusd

        We need to handle two messages:
        - spawn: add a new monitored child pid
        - reap: remove a killed child pid from monitoring
        """
        watcher_name, action, msg = self.split_data(data)
        logger.debug("received data from circusd: watcher.%s.%s, %s",
                     watcher_name, action, msg)
        # check if monitored watchers:
        if self._match_watcher_name(watcher_name):
            try:
                message = self.load_message(msg)
            except ValueError:
                logger.error("Error while decoding json for message: %s",
                             msg)
            else:
                if "process_pid" not in message:
                    logger.warning('no process_pid in message')
                    return
                pid = str(message.get("process_pid"))
                if action == "spawn":
                    self.pid_status[pid] = dict(watcher=watcher_name,
                                                last_activity=time.time())
                    logger.info("added new monitored pid for %s:%s",
                                watcher_name,
                                pid)
                # very questionable fix for Py3 here!
                # had to add check for pid in self.pid_status
                elif action == "reap" and pid in self.pid_status:
                    old_pid = self.pid_status.pop(pid)
                    logger.info("removed monitored pid for %s:%s",
                                old_pid['watcher'],
                                pid)
예제 #11
0
    def bind_and_listen(self):
        try:
            if self.is_unix:
                if os.path.exists(self.path):
                    raise OSError("%r already exists. You might want to "
                                  "remove it. If it's a stalled socket "
                                  "file, just restart Circus" % self.path)
                if self.umask is None:
                    self.bind(self.path)
                else:
                    old_mask = os.umask(self.umask)
                    self.bind(self.path)
                    os.umask(old_mask)
            else:
                if self.interface is not None:
                    # Bind to device if given, e.g. to limit which device to bind
                    # when binding on IN_ADDR_ANY or IN_ADDR_BROADCAST.
                    import IN
                    self.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE,
                                    self.interface + '\0')
                    logger.debug('Binding to device: %s' % self.interface)
                self.bind((self.host, self.port))
        except socket.error:
            logger.error('Could not bind %s' % self.location)
            raise

        self.setblocking(0)
        if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
            self.listen(self.backlog)

        if not self.is_unix:
            self.host, self.port = self.getsockname()

        logger.debug('Socket bound at %s - fd: %d' %
                     (self.location, self.fileno()))
예제 #12
0
파일: controller.py 프로젝트: atdt/circus
 def send_response(self, client_id, sock, msg, resp):
     try:
         sock.send(client_id, zmq.SNDMORE)
         sock.send(resp)
     except zmq.ZMQError as e:
         logger.error("Received %r - Could not send back %r - %s" %
                             (msg, resp, str(e)))
예제 #13
0
파일: controller.py 프로젝트: whitmo/circus
    def dispatch(self, job):
        cid, msg = job

        try:
            json_msg = json.loads(msg)
        except ValueError:
            return self.send_error(cid, msg, "json invalid",
                                   errno=errors.INVALID_JSON)

        cmd_name = json_msg.get('command')
        properties = json_msg.get('properties', {})
        cast = json_msg.get('msg_type') == "cast"

        try:
            if cmd_name is None:
                error = "no cmd: %r" % cmd_name
                return self.send_error(cid, msg, error, cast=cast,
                                       errno=errors.UNKNOWN_COMMAND)
            cmd = self.commands[cmd_name.lower()]
        except KeyError:
            error = "unknown command: %r" % cmd_name
            return self.send_error(cid, msg, error, cast=cast,
                                   errno=errors.UNKNOWN_COMMAND)

        try:
            cmd.validate(properties)
            resp = cmd.execute(self.arbiter, properties)
        except MessageError as e:
            return self.send_error(cid, msg, str(e), cast=cast,
                                   errno=errors.MESSAGE_ERROR)
        except OSError as e:
            return self.send_error(cid, msg, str(e), cast=cast,
                                   errno=errors.OS_ERROR)
        except:
            exctype, value = sys.exc_info()[:2]
            tb = traceback.format_exc()
            reason = "command %r: %s" % (msg, value)
            logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
            return self.send_error(cid, msg, reason, tb, cast=cast,
                                   errno=errors.COMMAND_ERROR)

        if resp is None:
            resp = ok()

        if not isinstance(resp, (dict, list,)):
            msg = "msg %r tried to send a non-dict: %s" % (msg, str(resp))
            logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
            return self.send_error(cid, msg, "server error", cast=cast,
                                   errno=errors.BAD_MSG_DATA_ERROR)

        if isinstance(resp, list):
            resp = {"results": resp}

        self.send_ok(cid, msg, resp, cast=cast)

        if cmd_name.lower() == "quit":
            if cid is not None:
                self.stream.flush()

            self.arbiter.stop()
예제 #14
0
파일: arbiter.py 프로젝트: zerok/circus
    def load_from_config(cls, config_file):
        cfg = get_config(config_file)

        watchers = []
        for watcher in cfg.get('watchers', []):
            watchers.append(Watcher.load_from_config(watcher))

        sockets = []
        for socket in cfg.get('sockets', []):
            sockets.append(CircusSocket.load_from_config(socket))

        httpd = cfg.get('httpd', False)
        if httpd:
            # controlling that we have what it takes to run the web UI
            # if something is missing this will tell the user
            try:
                import circusweb     # NOQA
            except ImportError:
                logger.error('You need to install circus-web')
                sys.exit(1)

        # creating arbiter
        arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
                      check_delay=cfg.get('check_delay', 1.),
                      prereload_fn=cfg.get('prereload_fn'),
                      stats_endpoint=cfg.get('stats_endpoint'),
                      plugins=cfg.get('plugins'), sockets=sockets,
                      warmup_delay=cfg.get('warmup_delay', 0),
                      httpd=httpd,
                      httpd_host=cfg.get('httpd_host', 'localhost'),
                      httpd_port=cfg.get('httpd_port', 8080),
                      debug=cfg.get('debug', False),
                      ssh_server=cfg.get('ssh_server', None))

        return arbiter
예제 #15
0
    def load_from_config(cls, config_file):
        cfg = get_config(config_file)

        watchers = []
        for watcher in cfg.get('watchers', []):
            watchers.append(Watcher.load_from_config(watcher))

        sockets = []
        for socket in cfg.get('sockets', []):
            sockets.append(CircusSocket.load_from_config(socket))

        httpd = cfg.get('httpd', False)
        if httpd:
            # controlling that we have what it takes to run the web UI
            # if something is missing this will tell the user
            try:
                import circusweb     # NOQA
            except ImportError:
                logger.error('You need to install circus-web')
                sys.exit(1)

        # creating arbiter
        arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
                      check_delay=cfg.get('check_delay', 1.),
                      prereload_fn=cfg.get('prereload_fn'),
                      stats_endpoint=cfg.get('stats_endpoint'),
                      plugins=cfg.get('plugins'), sockets=sockets,
                      warmup_delay=cfg.get('warmup_delay', 0),
                      httpd=httpd,
                      httpd_host=cfg.get('httpd_host', 'localhost'),
                      httpd_port=cfg.get('httpd_port', 8080),
                      debug=cfg.get('debug', False),
                      ssh_server=cfg.get('ssh_server', None))

        return arbiter
예제 #16
0
    def dispatch(self, job):
        cid, msg = job

        try:
            json_msg = json.loads(msg)
        except ValueError:
            return self.send_error(cid, msg, "json invalid",
                                   errno=errors.INVALID_JSON)

        cmd_name = json_msg.get('command')
        properties = json_msg.get('properties', {})
        cast = json_msg.get('msg_type') == "cast"

        try:
            cmd = self.commands[cmd_name.lower()]
        except KeyError:
            error = "unknown command: %r" % cmd_name
            return self.send_error(cid, msg, error, cast=cast,
                        errno=errors.UNKNOWN_COMMAND)

        try:
            cmd.validate(properties)
            resp = cmd.execute(self.arbiter, properties)
        except MessageError as e:
            return self.send_error(cid, msg, str(e), cast=cast,
                    errno=errors.MESSAGE_ERROR)
        except OSError as e:
            return self.send_error(cid, msg, str(e), cast=cast,
                    errno=errors.OS_ERROR)
        except:
            exctype, value = sys.exc_info()[:2]
            tb = traceback.format_exc()
            reason = "command %r: %s" % (msg, value)
            logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
            return self.send_error(cid, msg, reason, tb, cast=cast,
                    errno=errors.COMMAND_ERROR)

        if resp is None:
            resp = ok()

        if not isinstance(resp, (dict, list,)):
            msg = "msg %r tried to send a non-dict: %s" % (msg,
                    str(resp))
            logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
            return self.send_error(cid, msg, "server error", cast=cast,
                    errno=errors.BAD_MSG_DATA_ERROR)

        if isinstance(resp, list):
            resp = {"results": resp}

        self.send_ok(cid, msg, resp, cast=cast)

        if cmd_name.lower() == "quit":
            if cid is not None:
                self.stream.flush()

            self.arbiter.stop()
예제 #17
0
파일: util.py 프로젝트: PaulNendick/circus
def check_future_exception_and_log(future):
    if isinstance(future, concurrent.Future):
        exception = future.exception()
        if exception is not None:
            logger.error("exception %s caught" % exception)
            if hasattr(future, "exc_info"):
                exc_info = future.exc_info()
                traceback.print_tb(exc_info[2])
            return exception
예제 #18
0
def check_future_exception_and_log(future):
    if isinstance(future, concurrent.Future):
        exception = future.exception()
        if exception is not None:
            logger.error("exception %s caught" % exception)
            if hasattr(future, "exc_info"):
                exc_info = future.exc_info()
                traceback.print_tb(exc_info[2])
            return exception
def children_started(watcher, arbiter, hook_name, pid, **kwargs):
    name = watcher.name
    wid = watcher.processes[pid].wid
    try:
        wait_for_workers(name, wid, 'running', timeout_seconds=10,
                         minimum_age=5)
        return True
    except TimeoutError:
        logger.error('%s children are flapping on %d', name, pid)
        return False
def get_worker_states(name, wid, minimum_age=0.0):
    stats = get_uwsgi_stats(name, wid)
    if 'workers' not in stats:
        logger.error("Error: No workers found for WID %d of %d", wid, name)
        return ['unknown']
    workers = stats['workers']
    return [
        worker["status"] if 'status' in worker and worker['last_spawn'] < time() - minimum_age else 'unknown'
        for worker in workers
    ]
예제 #21
0
def get_worker_states(name, wid, base_port, minimum_age=0.0):
    stats = get_uwsgi_stats(name, wid, base_port)
    if 'workers' not in stats:
        logger.error("Error: No workers found for WID %d of %d", wid, name)
        return ['unknown']
    workers = stats['workers']
    return [
        worker["status"] if 'status' in worker
        and worker['last_spawn'] < time() - minimum_age else 'unknown'
        for worker in workers
    ]
예제 #22
0
    def load_from_config(cls, config_file, loop=None):
        cfg = get_config(config_file)
        watchers = []
        for watcher in cfg.get('watchers', []):
            watchers.append(Watcher.load_from_config(watcher))

        sockets = []
        for socket_ in cfg.get('sockets', []):
            sockets.append(CircusSocket.load_from_config(socket_))

        httpd = cfg.get('httpd', False)
        if httpd:
            # controlling that we have what it takes to run the web UI
            # if something is missing this will tell the user
            try:
                import circusweb  # NOQA
            except ImportError:
                logger.error('You need to install circus-web')
                sys.exit(1)

        # creating arbiter
        arbiter = cls(watchers,
                      cfg['endpoint'],
                      cfg['pubsub_endpoint'],
                      check_delay=cfg.get('check_delay', 1.),
                      prereload_fn=cfg.get('prereload_fn'),
                      statsd=cfg.get('statsd', False),
                      stats_endpoint=cfg.get('stats_endpoint'),
                      papa_endpoint=cfg.get('papa_endpoint'),
                      multicast_endpoint=cfg.get('multicast_endpoint'),
                      plugins=cfg.get('plugins'),
                      sockets=sockets,
                      warmup_delay=cfg.get('warmup_delay', 0),
                      httpd=httpd,
                      loop=loop,
                      httpd_host=cfg.get('httpd_host', 'localhost'),
                      httpd_port=cfg.get('httpd_port', 8080),
                      debug=cfg.get('debug', False),
                      debug_gc=cfg.get('debug_gc', False),
                      ssh_server=cfg.get('ssh_server', None),
                      pidfile=cfg.get('pidfile', None),
                      loglevel=cfg.get('loglevel', None),
                      logoutput=cfg.get('logoutput', None),
                      loggerconfig=cfg.get('loggerconfig', None),
                      fqdn_prefix=cfg.get('fqdn_prefix', None),
                      umask=cfg['umask'],
                      endpoint_owner=cfg.get('endpoint_owner', None))

        # store the cfg which will be used, so it can be used later
        # for checking if the cfg has been changed
        arbiter._cfg = cls.get_arbiter_config(cfg)
        arbiter.config_file = config_file

        return arbiter
예제 #23
0
파일: sockets.py 프로젝트: j4mie/circus
    def bind_and_listen(self):
        try:
            self.bind((self.host, self.port))
        except socket.error:
            logger.error('Could not bind %s:%d' % (self.host, self.port))
            raise

        self.setblocking(0)
        self.listen(self.backlog)
        self.host, self.port = self.getsockname()
        logger.debug('Socket bound at %s:%d - fd: %d' % (self.host, self.port,
                                                         self.fileno()))
예제 #24
0
파일: sighandler.py 프로젝트: themgt/circus
 def signal(self, sig, frame):
     signame = self.SIG_NAMES.get(sig)
     if signame is not None:
         try:
             handler = getattr(self, "handle_%s" % signame)
             handler()
         except AttributeError:
             pass
         except Exception as e:
             tb = traceback.format_exc()
             logger.error("error: %s [%s]" % (e, tb))
             sys.exit(1)
예제 #25
0
 def signal(self, sig, frame):
     signame = self.SIG_NAMES.get(sig)
     if signame is not None:
         try:
             handler = getattr(self, "handle_%s" % signame)
             handler()
         except AttributeError:
             pass
         except Exception, e:
             tb = traceback.format_exc()
             logger.error("error: %s [%s]" % (e, tb))
             sys.exit(1)
예제 #26
0
파일: watcher.py 프로젝트: jsbronder/circus
    def reap_process(self, pid, status=None):
        """ensure that the process is killed (and not a zombie)"""
        if pid not in self.processes:
            return
        process = self.processes.pop(pid)

        if status is None:
            while True:
                try:
                    _, status = os.waitpid(pid, os.WNOHANG)
                except OSError as e:
                    if e.errno == errno.EAGAIN:
                        time.sleep(0.001)
                        continue
                    elif e.errno == errno.ECHILD:
                        # nothing to do here, we do not have any child
                        # process running
                        # but we still need to send the "reap" signal.
                        #
                        # This can happen if poll() or wait() were called on
                        # the underlying process.
                        logger.error('reaping already dead process %s [%s]',
                                     pid, self.name)
                        self.notify_event(
                            "reap",
                            {"process_pid": pid,
                             "time": time.time(),
                             "exit_code": process.returncode()})
                        process.stop()
                        return
                    else:
                        raise

        # get return code
        if os.WIFSIGNALED(status):
            exit_code = os.WTERMSIG(status)
        # process exited using exit(2) system call; return the
        # integer exit(2) system call has been called with
        elif os.WIFEXITED(status):
            exit_code = os.WEXITSTATUS(status)
        else:
            # should never happen
            raise RuntimeError("Unknown process exit status")

        # if the process is dead or a zombie try to definitely stop it.
        if process.status in (DEAD_OR_ZOMBIE, UNEXISTING):
            process.stop()

        logger.debug('reaping process %s [%s]', pid, self.name)
        self.notify_event("reap",
                          {"process_pid": pid,
                           "time": time.time(),
                           "exit_code": exit_code})
예제 #27
0
    def load_from_config(cls, config_file, loop=None):
        cfg = get_config(config_file)
        watchers = []
        for watcher in cfg.get("watchers", []):
            watchers.append(Watcher.load_from_config(watcher))

        sockets = []
        for socket in cfg.get("sockets", []):
            sockets.append(CircusSocket.load_from_config(socket))

        httpd = cfg.get("httpd", False)
        if httpd:
            # controlling that we have what it takes to run the web UI
            # if something is missing this will tell the user
            try:
                import circusweb  # NOQA
            except ImportError:
                logger.error("You need to install circus-web")
                sys.exit(1)

        # creating arbiter
        arbiter = cls(
            watchers,
            cfg["endpoint"],
            cfg["pubsub_endpoint"],
            check_delay=cfg.get("check_delay", 1.0),
            prereload_fn=cfg.get("prereload_fn"),
            statsd=cfg.get("statsd", False),
            stats_endpoint=cfg.get("stats_endpoint"),
            multicast_endpoint=cfg.get("multicast_endpoint"),
            plugins=cfg.get("plugins"),
            sockets=sockets,
            warmup_delay=cfg.get("warmup_delay", 0),
            httpd=httpd,
            loop=loop,
            httpd_host=cfg.get("httpd_host", "localhost"),
            httpd_port=cfg.get("httpd_port", 8080),
            debug=cfg.get("debug", False),
            ssh_server=cfg.get("ssh_server", None),
            pidfile=cfg.get("pidfile", None),
            loglevel=cfg.get("loglevel", None),
            logoutput=cfg.get("logoutput", None),
            fqdn_prefix=cfg.get("fqdn_prefix", None),
            umask=cfg["umask"],
        )

        # store the cfg which will be used, so it can be used later
        # for checking if the cfg has been changed
        arbiter._cfg = cls.get_arbiter_config(cfg)
        arbiter.config_file = config_file

        return arbiter
예제 #28
0
def children_started(watcher, arbiter, hook_name, pid, **kwargs):
    name = watcher.name
    wid = watcher.processes[pid].wid
    try:
        wait_for_workers(name,
                         wid,
                         'running',
                         timeout_seconds=10,
                         minimum_age=5)
        return True
    except TimeoutError:
        logger.error('%s children are flapping on %d', name, pid)
        return False
예제 #29
0
    def bind_and_listen(self):
        try:
            self.bind((self.host, self.port))
        except socket.error:
            logger.error('Could not bind %s:%d' % (self.host, self.port))
            raise

        self.setblocking(0)
        if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
            self.listen(self.backlog)
        self.host, self.port = self.getsockname()
        logger.debug('Socket bound at %s:%d - fd: %d' %
                     (self.host, self.port, self.fileno()))
예제 #30
0
    def load_from_config(cls, config_file, loop=None):
        cfg = get_config(config_file)
        watchers = []
        for watcher in cfg.get('watchers', []):
            watchers.append(Watcher.load_from_config(watcher))

        sockets = []
        for socket_ in cfg.get('sockets', []):
            sockets.append(CircusSocket.load_from_config(socket_))

        httpd = cfg.get('httpd', False)
        if httpd:
            # controlling that we have what it takes to run the web UI
            # if something is missing this will tell the user
            try:
                import circusweb  # NOQA
            except ImportError:
                logger.error('You need to install circus-web')
                sys.exit(1)

        # creating arbiter
        arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'],
                      check_delay=cfg.get('check_delay', 1.),
                      prereload_fn=cfg.get('prereload_fn'),
                      statsd=cfg.get('statsd', False),
                      stats_endpoint=cfg.get('stats_endpoint'),
                      papa_endpoint=cfg.get('papa_endpoint'),
                      multicast_endpoint=cfg.get('multicast_endpoint'),
                      plugins=cfg.get('plugins'), sockets=sockets,
                      warmup_delay=cfg.get('warmup_delay', 0),
                      httpd=httpd,
                      loop=loop,
                      httpd_host=cfg.get('httpd_host', 'localhost'),
                      httpd_port=cfg.get('httpd_port', 8080),
                      debug=cfg.get('debug', False),
                      debug_gc=cfg.get('debug_gc', False),
                      ssh_server=cfg.get('ssh_server', None),
                      pidfile=cfg.get('pidfile', None),
                      loglevel=cfg.get('loglevel', None),
                      logoutput=cfg.get('logoutput', None),
                      loggerconfig=cfg.get('loggerconfig', None),
                      fqdn_prefix=cfg.get('fqdn_prefix', None),
                      umask=cfg['umask'],
                      endpoint_owner=cfg.get('endpoint_owner', None))

        # store the cfg which will be used, so it can be used later
        # for checking if the cfg has been changed
        arbiter._cfg = cls.get_arbiter_config(cfg)
        arbiter.config_file = config_file

        return arbiter
예제 #31
0
    def dispatch(self, job):
        cid, msg = job

        try:
            json_msg = json.loads(msg)
        except ValueError:
            return self.send_error(cid, msg, "json invalid")

        cmd_name = json_msg.get('command')
        properties = json_msg.get('properties', {})

        try:
            cmd = self.commands[cmd_name.lower()]
        except KeyError:
            error = "unknown command: %r" % cmd_name
            return self.send_error(cid, msg, error)

        try:
            cmd.validate(properties)
            resp = cmd.execute(self.arbiter, properties)
        except MessageError as e:
            return self.send_error(cid, msg, str(e))
        except OSError as e:
            return self.send_error(cid, msg, str(e))
        except:
            exctype, value = sys.exc_info()[:2]
            tb = traceback.format_exc()
            reason = "command %r: %s" % (msg, value)
            logger.debug("error: command %r: %s\n\n%s", msg, value, tb)
            return self.send_error(cid, msg, reason, tb)

        if resp is None:
            resp = ok()

        if not isinstance(resp, (dict, list,)):
            msg = "msg %r tried to send a non-dict: %s" % (msg,
                    str(resp))
            logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
            return self.send_error(cid, msg, "server error")

        if isinstance(resp, list):
            resp = {"results": resp}

        self.send_ok(cid, msg, resp)

        if cmd_name.lower() == "quit":
            if cid is not None:
                self.stream.flush()

            self.arbiter.stop()
예제 #32
0
def get_uwsgi_stats(name, wid, base_port):
    sock = socket.create_connection(('127.0.0.1', base_port + wid), timeout=1)
    received = sock.recv(100000)
    data = bytes()
    while received:
        data += received
        received = sock.recv(100000)
    if not data:
        logger.error("Error: No stats seem available for WID %d of %s", wid,
                     name)
        return
    # recent versions of uWSGI had some garbage in the JSON so strip it out
    data = data.decode('latin', 'replace')
    data = NON_JSON_CHARACTERS.sub('', data)
    return loads(data)
예제 #33
0
def children_started(watcher, arbiter, hook_name, pid, **kwargs):
    name = watcher.name
    wid = watcher.processes[pid].wid
    base_port = int(watcher._options.get('stats_base_port', 8090))
    logger.info('%s waiting for workers', name)
    try:
        wait_for_workers(name, wid, base_port, 'running', timeout_seconds=15,
                         minimum_age=5)
        return True
    except TimeoutError:
        logger.error('%s children are flapping on %d', name, pid)
        return False
    except Exception:
        logger.error('%s not publishing stats on %d', name, pid)
        return False
예제 #34
0
def get_uwsgi_stats(name, wid, base_port):
    sock = socket.create_connection(('127.0.0.1', base_port + wid), timeout=1)
    received = sock.recv(100000)
    data = bytes()
    while received:
        data += received
        received = sock.recv(100000)
    if not data:
        logger.error(
            "Error: No stats seem available for WID %d of %s", wid, name)
        return
    # recent versions of uWSGI had some garbage in the JSON so strip it out
    if not PY2:
        data = data.decode('latin', 'replace')
    data = NON_JSON_CHARACTERS.sub('', data)
    return loads(data)
예제 #35
0
    def bind_and_listen(self):
        try:
            if self.is_unix:
                if os.path.exists(self.path):
                    if self.replace:
                        os.unlink(self.path)
                    else:
                        raise OSError(
                            "%r already exists. You might want to "
                            "remove it. If it's a stalled socket "
                            "file, just restart Circus" % self.path
                        )
                if self.umask is None:
                    self.bind(self.path)
                else:
                    old_mask = os.umask(self.umask)
                    self.bind(self.path)
                    os.umask(old_mask)
            else:
                if self.interface is not None:
                    # Bind to device if given, e.g. to limit which device to
                    # bind when binding on IN_ADDR_ANY or IN_ADDR_BROADCAST.
                    import IN

                    if hasattr(IN, "SO_BINDTODEVICE"):
                        self.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE, self.interface + "\0")
                        logger.debug("Binding to device: %s" % self.interface)

                self.bind((self.host, self.port))
        except socket.error:
            logger.error("Could not bind %s" % self.location)
            raise

        if self.blocking:
            self.setblocking(1)
        else:
            self.setblocking(0)
        if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
            self.listen(self.backlog)

        if not self.is_unix:
            if self.family == socket.AF_INET6:
                self.host, self.port, _flowinfo, _scopeid = self.getsockname()
            else:
                self.host, self.port = self.getsockname()

        logger.debug("Socket bound at %s - fd: %d" % (self.location, self.fileno()))
예제 #36
0
파일: plugin.py 프로젝트: amol-/vhostino
    def _setup_vhost(self, sockets, worker):
        options = self.call('options', name=worker)
        if not options or options.get('status') != 'ok':
            logger.error('Failed to retrieve options')
            return

        options = options['options']
        vhost = options.get('vhostino.vhost')
        #logger.info('\n\nCONF %s -> %s', worker, options)
        if vhost == 'True':
            socket = sockets.get(worker)
            if socket:
                logger.info('Vhostino: Registering %s -> %s', worker, socket['port'])
                self.server.config.add_vhost(worker, socket['port'])
                if options.get('vhostino.default_vhost'):
                    logger.info('Vhostino: Default Vhost %s -> %s', worker, socket['port'])
                    self.server.config.set_default(socket['port'])
예제 #37
0
파일: plugin.py 프로젝트: amol-/vhostino
    def _initialize_proxy(self):
        sockets = self.call('listsockets')
        if not sockets or sockets.get('status') != 'ok':
            logger.error('Failed to retrieve existing sockets list')
            return
        sockets = self._sockets_by_name(sockets['sockets'])

        status = self.call('status')
        if not status and status.get('status') != 'ok':
            logger.error('Failed to retrieve status')
            return

        for worker in status['statuses']:
            self._setup_vhost(sockets, worker)

        logger.info('VHosting: Serving on %d', self.server.server_port)
        self.server.serve_forever()
예제 #38
0
    def _dispatch_callback(self, msg, cid, mid, cast, cmd_name, resp=None):
        if resp is None:
            resp = ok()

        if not isinstance(resp, (dict, list)):
            msg = "msg %r tried to send a non-dict: %s" % (msg, str(resp))
            logger.error("msg %r tried to send a non-dict: %s", msg, str(resp))
            return self.send_error(mid, cid, msg, "server error", cast=cast,
                                   errno=errors.BAD_MSG_DATA_ERROR)

        if isinstance(resp, list):
            resp = {"results": resp}

        self.send_ok(mid, cid, msg, resp, cast=cast)

        if cmd_name.lower() == "quit":
            if cid is not None:
                self.stream.flush()
예제 #39
0
def clean_stop(watcher, arbiter, hook_name, pid, signum, **kwargs):
    if len(watcher.processes) > watcher.numprocesses and signum == signal.SIGQUIT:
        name = watcher.name
        started = watcher.processes[pid].started
        newer_pids = [p for p, w in watcher.processes.items() if p != pid and w.started > started]
        # if the one being stopped is actually the newer one, just do it
        if len(newer_pids) < watcher.numprocesses:
            return True
        wid = watcher.processes[pid].wid
        base_port = int(watcher._options.get('stats_base_port', 8090))
        logger.info('%s pausing', name)
        try:
            watcher.send_signal(pid, signal.SIGTSTP)
            wait_for_workers(name, wid, base_port, 'paused')
            logger.info('%s workers idle', name)
        except Exception as e:
            logger.error('trouble pausing %s: %s', name, e)
    return True
예제 #40
0
파일: watchdog.py 프로젝트: suligap/circus
 def _bind_socket(self):
     """bind the listening socket for watchdog udp and start an event
     handler for handling udp received messages.
     """
     self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     try:
         self.sock.bind((self.watchdog_ip, self.watchdog_port))
     except socket.error as socket_error:
         logger.error(
             "Problem while binding watchdog socket on %s:%s (err %s",
             self.watchdog_ip, self.watchdog_port, str(socket_error))
         self.sock = None
     else:
         self.sock.settimeout(1)
         self.loop.add_handler(self.sock.fileno(), self.receive_udp_socket,
                               ioloop.IOLoop.READ)
         logger.info("Watchdog listening UDP on %s:%s", self.watchdog_ip,
                     self.watchdog_port)
예제 #41
0
파일: sockets.py 프로젝트: santagada/circus
    def bind_and_listen(self):
        try:
            if self.is_unix:
                self.bind(self.path)
            else:
                self.bind((self.host, self.port))
        except socket.error:
            logger.error('Could not bind %s' % self.location)
            raise

        self.setblocking(0)
        if self.socktype in (socket.SOCK_STREAM, socket.SOCK_SEQPACKET):
            self.listen(self.backlog)

        if not self.is_unix:
            self.host, self.port = self.getsockname()

        logger.debug('Socket bound at %s - fd: %d' % (self.location,
                                                      self.fileno()))
예제 #42
0
파일: watchdog.py 프로젝트: kyudin/circus
 def _bind_socket(self):
     """bind the listening socket for watchdog udp and start an event
     handler for handling udp received messages.
     """
     self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     try:
         self.sock.bind((self.watchdog_ip, self.watchdog_port))
     except socket.error as socket_error:
         logger.error(
             "Problem while binding watchdog socket on %s:%s (err %s",
             self.watchdog_ip,
             self.watchdog_port,
             str(socket_error),
         )
         self.sock = None
     else:
         self.sock.settimeout(1)
         self.loop.add_handler(self.sock.fileno(), self.receive_udp_socket, ioloop.IOLoop.READ)
         logger.info("Watchdog listening UDP on %s:%s", self.watchdog_ip, self.watchdog_port)
예제 #43
0
def children_started(watcher, arbiter, hook_name, pid, **kwargs):
    name = watcher.name
    wid = watcher.processes[pid].wid
    base_port = int(watcher._options.get('stats_base_port', 8090))
    logger.info('%s waiting for workers', name)
    try:
        wait_for_workers(name,
                         wid,
                         base_port,
                         'running',
                         timeout_seconds=15,
                         minimum_age=5)
        return True
    except TimeoutError:
        logger.error('%s children are flapping on %d', name, pid)
        return False
    except Exception:
        logger.error('%s not publishing stats on %d', name, pid)
        return False
예제 #44
0
def get_uwsgi_stats(name, wid):
    try:
        sock = socket.create_connection(('127.0.0.1', 8090 + wid), timeout=1)
    except Exception as e:
        logger.error(
            "Error: Connection refused for {0}} on 127.0.0.1:809{1} - {2}".
            format(name, wid, e))
        raise e
    received = sock.recv(100000)
    data = bytes()
    while received:
        data += received
        received = sock.recv(100000)
    if not data:
        logger.error("Error: No stats seem available for WID %d of %s", wid,
                     name)
        return
    # recent versions of uWSGI had some garbage in the JSON so strip it out
    data = NON_JSON_CHARACTERS.sub('', data.decode())
    return loads(data)
def get_uwsgi_stats(name, wid):
    try:
        sock = socket.create_connection(('127.0.0.1', 8090 + wid),
                                        timeout=1)
    except Exception as e:
        logger.error(
            "Error: Connection refused for {0}} on 127.0.0.1:809{1} - {2}"
            .format(name, wid, e))
        raise e
    received = sock.recv(100000)
    data = bytes()
    while received:
        data += received
        received = sock.recv(100000)
    if not data:
        logger.error(
            "Error: No stats seem available for WID %d of %s", wid, name)
        return
    # recent versions of uWSGI had some garbage in the JSON so strip it out
    data = NON_JSON_CHARACTERS.sub('', data.decode())
    return loads(data)
예제 #46
0
def clean_stop(watcher, arbiter, hook_name, pid, signum, **kwargs):
    if len(watcher.processes
           ) > watcher.numprocesses and signum == signal.SIGQUIT:
        name = watcher.name
        started = watcher.processes[pid].started
        newer_pids = [
            p for p, w in watcher.processes.items()
            if p != pid and w.started > started
        ]
        # if the one being stopped is actually the newer one, just do it
        if len(newer_pids) < watcher.numprocesses:
            return True
        wid = watcher.processes[pid].wid
        base_port = int(watcher._options.get('stats_base_port', 8090))
        logger.info('%s pausing', name)
        try:
            watcher.send_signal(pid, signal.SIGTSTP)
            wait_for_workers(name, wid, base_port, 'paused')
            logger.info('%s workers idle', name)
        except Exception as e:
            logger.error('trouble pausing %s: %s', name, e)
    return True
예제 #47
0
 def handle_callback_exception(callback):
     logger.error("Exception in callback %r", callback, exc_info=True)