Ejemplo n.º 1
0
 def __init__(self, pidfile, socketfile, config, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
     Daemon.__init__(self, pidfile, stdin, stdout, stderr)
     self.__running=True
     self.__socket_file=socketfile
     self.config=config
     self.__outputs={}
     self.__pusher_pid=-1
     self.__tailer=None
     self.__buffer_thread=None
     self.__collectors=[]
     self.__asyncServer = AsyncServer(self)
Ejemplo n.º 2
0
 def __init__(self, pidfile, socketfile, config, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
     Daemon.__init__(self, pidfile, stdin, stdout, stderr)
     self.__running=True
     self.__socket_file=socketfile
     self.config=config
     self.__outputs={}
     self.__pusher_pid=-1
     self.__tailer = None
     self.__buffer_thread=None
     self.__collectors=[]
     self.__asyncServer = AsyncServer(self)
Ejemplo n.º 3
0
class ProducerDaemon(Daemon):
    def __init__(self,
                 pidfile,
                 socketfile,
                 config,
                 stdin='/dev/null',
                 stdout='/dev/null',
                 stderr='/dev/null'):
        Daemon.__init__(self, pidfile, stdin, stdout, stderr)
        self.__running = True
        self.__socket_file = socketfile
        self.config = config
        self.__outputs = {}
        self.__pusher_pid = -1
        self.__tailer = None
        self.__buffer_thread = None
        self.__collectors = []
        self.__asyncServer = AsyncServer(self)

    def __sigTERMhandler(self, signum, frame):
        log.debug("Caught signal %d. Exiting" % signum)
        self.quit()

    def quit(self):
        self.__asyncServer.stop()
        for c in self.__collectors:
            c.quit()
        if self.__buffer_thread is not None:
            self.__buffer_thread.quit()
        if self.__pusher_pid > -1:
            try:
                os.kill(self.__pusher_pid, signal.SIGTERM)
            except OSError as err:
                err = str(err)
                if err.find("No such process") > 0:
                    log.info("Pusher process has gone.")
        self.__running = False

    def init(self):
        """Set up all necessary directories and etc"""
        if 'global' in config:
            global_vars = config['global']
            set_global(global_vars)

        for n, cfg in config['output'].iteritems():
            if n == 'buffer':
                if not 'directory' in cfg:
                    print "ERROR: buffer directory not specified in config."
                    return False
                buffer_dir = cfg['directory']
                if os.path.exists(buffer_dir) and (
                        not os.path.isdir(buffer_dir)):
                    print "ERROR: buffer directory exists but it is not a directory."
                    return False
                if not os.path.exists(buffer_dir):
                    log.info("Creating buffer directory %s.", buffer_dir)
                    os.makedirs(buffer_dir)
                self.__outputs[n] = BufferOutput(cfg)
            elif n == 'kafka-http':
                self.__outputs[n] = KafkaHTTPOutput(cfg)
            elif n == 'aws':
                self.__outputs[n] = AWSOutput(cfg)
            elif n == 'file':
                self.__outputs[n] = FileOutput(cfg)
            elif 'class' in cfg:
                arguments = {}
                if 'arguments' in cfg:
                    arguments = cfg['arguments']
                self.__outputs[n] = init_object(cfg['class'], **arguments)

        if 'pusher' in config:
            if not 'directory' in config['pusher'] or not 'output' in config[
                    'pusher']:
                print "ERROR: need to specify directory and output in pusher."
                return False

        if 'tailer' in config:
            self.__tailer = Tailer(config['tailer'])

        return True

    def run(self):
        # Install signal handlers
        signal.signal(signal.SIGTERM, self.__sigTERMhandler)
        signal.signal(signal.SIGINT, self.__sigTERMhandler)
        # Ensure unhandled exceptions are logged
        sys.excepthook = excepthook
        log.info("Reporting producer started at %s",
                 datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y"))
        if 'pusher' in config:
            pusher = Pusher(self.__outputs[config['pusher']['output']],
                            config['pusher']['directory'],
                            config['pusher'].get('batch', 1),
                            config['pusher'].get('stats_on', False),
                            config['pusher'].get('back_off_indicator', None))
            pusher.daemon = True
            pusher.start()
            self.__pusher_pid = pusher.pid
            log.info("started pusher pid=%d", pusher.pid)
        if 'buffer' in self.__outputs:
            self.__buffer_thread = BufferThread(self.__outputs['buffer'])
        if 'collector' in config:
            for collector_config in config['collector']:
                log.debug("Initiating collector %s", collector_config)
                log.debug("Output object: %s", self.__outputs)
                c = Collector(
                    collector_config, config['collector'][collector_config],
                    self.__outputs[config['collector'][collector_config]
                                   ['output']], self.__tailer)
                self.__collectors.append(c)
        for c in self.__collectors:
            c.start()
        if self.__buffer_thread is not None:
            log.info("starting buffer thread")
            self.__buffer_thread.start()
        # Start the communication
        log.debug("Starting console communication")
        try:
            self.__asyncServer.start(self.__socket_file)
        except AsyncServerException as e:
            log.exception("Could not start socket server: %s", e)
        if 'pusher' in config:
            pusher.join()
        if self.__buffer_thread is not None and self.__buffer_thread.is_alive(
        ):
            self.__buffer_thread.join()
        for c in self.__collectors:
            if c.is_alive():
                c.join()
        log.info("Reporting producer stopped at %s",
                 datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y"))

    def console(self, message):
        if message == "config":
            return json.dumps({"producer-config": self.config})
        elif message == "show":
            return json.dumps(
                {"collectors": [c.info() for c in self.__collectors]})
        elif message == "check":
            result = []
            if 'buffer' in self.__outputs:
                if not self.__outputs['buffer'].check_dir.is_ok():
                    result.append("Buffer directory %s is full." %
                                  self.__outputs['buffer'].directory)
            for c in self.__collectors:
                info = c.info()
                if not info['is_running']:
                    result.append("Collector %s has stopped." % info['name'])
            return json.dumps({"check-result": result})
        elif message == "help":
            return "{\"system\": \"Command list: config, show, check\"}"
        else:
            return "{\"system\": \"Unknown command. Please run help to get a list of supported commands.\"}"
Ejemplo n.º 4
0
class ProducerDaemon(Daemon):
    def __init__(self, pidfile, socketfile, config, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
        Daemon.__init__(self, pidfile, stdin, stdout, stderr)
        self.__running=True
        self.__socket_file=socketfile
        self.config=config
        self.__outputs={}
        self.__pusher_pid=-1
        self.__tailer = None
        self.__buffer_thread=None
        self.__collectors=[]
        self.__asyncServer = AsyncServer(self)

    def __sigTERMhandler(self, signum, frame):
        log.debug("Caught signal %d. Exiting" % signum)
        self.quit()

    def quit(self):
        self.__asyncServer.stop()
        for c in self.__collectors:
            c.quit()
        if self.__buffer_thread is not None:
            self.__buffer_thread.quit()
        if self.__pusher_pid>-1:
            try:
                os.kill(self.__pusher_pid, signal.SIGTERM)
            except OSError as err:
                err = str(err)
                if err.find("No such process") > 0:
                    log.info("Pusher process has gone.")
        self.__running=False

    def init(self):
        """Set up all necessary directories and etc"""
        if 'global' in config:
            global_vars = config['global']
            set_global(global_vars)

        for n, cfg in config['output'].iteritems():
            if n == 'buffer':
                if not 'directory' in cfg:
                    print "ERROR: buffer directory not specified in config."
                    return False
                buffer_dir = cfg['directory']
                if os.path.exists(buffer_dir) and (not os.path.isdir(buffer_dir)):
                    print "ERROR: buffer directory exists but it is not a directory."
                    return False
                if not os.path.exists(buffer_dir):
                    log.info("Creating buffer directory %s.", buffer_dir)
                    os.makedirs(buffer_dir)
                self.__outputs[n] = BufferOutput(cfg)
            elif n == 'kafka-http':
                self.__outputs[n] = KafkaHTTPOutput(cfg)
            elif n == 'file':
                self.__outputs[n] = FileOutput(cfg)
            elif 'class' in cfg:
                arguments = {}
                if 'arguments' in cfg:
                    arguments = cfg['arguments']
                self.__outputs[n] = init_object(cfg['class'], **arguments)

        if 'pusher' in config:
            if not 'directory' in config['pusher'] or not 'output' in config['pusher']:
                print "ERROR: need to specify directory and output in pusher."
                return False

        if 'tailer' in config:
            self.__tailer = Tailer(config['tailer'])

        return True

    def run(self):
        # Install signal handlers
        signal.signal(signal.SIGTERM, self.__sigTERMhandler)
        signal.signal(signal.SIGINT, self.__sigTERMhandler)
        # Ensure unhandled exceptions are logged
        sys.excepthook = excepthook
        log.info("Reporting producer started at %s",
                 datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y"))
        if 'pusher' in config:
            pusher = Pusher(self.__outputs[config['pusher']['output']],
                            config['pusher']['directory'],
                            config['pusher'].get('batch', 1),
                            config['pusher'].get('stats_on', False),
                            config['pusher'].get('back_off_indicator', None))
            pusher.daemon = True
            pusher.start()
            self.__pusher_pid = pusher.pid
            log.info("started pusher pid=%d", pusher.pid)
        if 'buffer' in self.__outputs:
            self.__buffer_thread = BufferThread(self.__outputs['buffer'])
        if 'collector' in config:
            for collector_config in config['collector']:
                log.debug("Initiating collector %s", collector_config)
                log.debug("self.__outputs: %s", self.__outputs)
                c = Collector(collector_config, config['collector'][collector_config],
                              self.__outputs[config['collector'][collector_config]['output']],
                              self.__tailer)
                self.__collectors.append(c)
        for c in self.__collectors:
            c.start()
        if self.__buffer_thread is not None:
            log.info("starting buffer thread")
            self.__buffer_thread.start()
        # Start the communication
        log.debug("Starting console communication")
        try:
            self.__asyncServer.start(self.__socket_file)
        except AsyncServerException as e:
            log.exception("Could not start socket server: %s", e)
        if 'pusher' in config:
            pusher.join()
        if self.__buffer_thread is not None and self.__buffer_thread.is_alive():
            self.__buffer_thread.join()
        for c in self.__collectors:
            if c.is_alive():
                c.join()
        log.info("Reporting producer stopped at %s",
                 datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y"))

    def console(self, message):
        if message=="config":
            return json.dumps({"producer-config":self.config})
        elif message=="show":
            return json.dumps({"collectors": [c.info() for c in self.__collectors]})
        elif message=="check":
            result=[]
            if 'buffer' in self.__outputs:
                if not self.__outputs['buffer'].check_dir.is_ok():
                    result.append("Buffer directory %s is full." % self.__outputs['buffer'].directory)
            for c in self.__collectors:
                info=c.info()
                if not info['is_running']:
                    result.append("Collector %s has stopped." % info['name'])
            return json.dumps({"check-result":result})
        elif message=="help":
            return "{\"system\": \"Command list: config, show, check\"}"
        else:
            return "{\"system\": \"Unknown command. Please run help to get a list of supported commands.\"}"