def run(self): # Install signal handlers signal.signal(signal.SIGTERM, self.__sigTERMhandler) signal.signal(signal.SIGINT, self.__sigTERMhandler) # Ensure unhandled exceptions are logged sys.excepthook = excepthook log.info("Reporting producer started at %s", datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")) if 'pusher' in config: pusher = Pusher(self.__outputs[config['pusher']['output']], config['pusher']['directory'], config['pusher'].get('batch', 1), config['pusher'].get('stats_on', False), config['pusher'].get('back_off_indicator', None)) pusher.daemon = True pusher.start() self.__pusher_pid = pusher.pid log.info("started pusher pid=%d", pusher.pid) if 'buffer' in self.__outputs: self.__buffer_thread = BufferThread(self.__outputs['buffer']) if 'collector' in config: for collector_config in config['collector']: log.debug("Initiating collector %s", collector_config) log.debug("Output object: %s", self.__outputs) c = Collector( collector_config, config['collector'][collector_config], self.__outputs[config['collector'][collector_config] ['output']], self.__tailer) self.__collectors.append(c) for c in self.__collectors: c.start() if self.__buffer_thread is not None: log.info("starting buffer thread") self.__buffer_thread.start() # Start the communication log.debug("Starting console communication") try: self.__asyncServer.start(self.__socket_file) except AsyncServerException as e: log.exception("Could not start socket server: %s", e) if 'pusher' in config: pusher.join() if self.__buffer_thread is not None and self.__buffer_thread.is_alive( ): self.__buffer_thread.join() for c in self.__collectors: if c.is_alive(): c.join() log.info("Reporting producer stopped at %s", datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y"))
def run(self): # Install signal handlers signal.signal(signal.SIGTERM, self.__sigTERMhandler) signal.signal(signal.SIGINT, self.__sigTERMhandler) # Ensure unhandled exceptions are logged sys.excepthook = excepthook log.info("Reporting producer started at %s", datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")) if 'pusher' in config: pusher = Pusher(self.__outputs[config['pusher']['output']], config['pusher']['directory'], config['pusher'].get('batch', 1), config['pusher'].get('stats_on', False), config['pusher'].get('back_off_indicator', None)) pusher.daemon = True pusher.start() self.__pusher_pid = pusher.pid log.info("started pusher pid=%d", pusher.pid) if 'buffer' in self.__outputs: self.__buffer_thread = BufferThread(self.__outputs['buffer']) if 'collector' in config: for collector_config in config['collector']: log.debug("Initiating collector %s", collector_config) log.debug("self.__outputs: %s", self.__outputs) c = Collector(collector_config, config['collector'][collector_config], self.__outputs[config['collector'][collector_config]['output']], self.__tailer) self.__collectors.append(c) for c in self.__collectors: c.start() if self.__buffer_thread is not None: log.info("starting buffer thread") self.__buffer_thread.start() # Start the communication log.debug("Starting console communication") try: self.__asyncServer.start(self.__socket_file) except AsyncServerException as e: log.exception("Could not start socket server: %s", e) if 'pusher' in config: pusher.join() if self.__buffer_thread is not None and self.__buffer_thread.is_alive(): self.__buffer_thread.join() for c in self.__collectors: if c.is_alive(): c.join() log.info("Reporting producer stopped at %s", datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y"))
class ProducerDaemon(Daemon): def __init__(self, pidfile, socketfile, config, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): Daemon.__init__(self, pidfile, stdin, stdout, stderr) self.__running = True self.__socket_file = socketfile self.config = config self.__outputs = {} self.__pusher_pid = -1 self.__tailer = None self.__buffer_thread = None self.__collectors = [] self.__asyncServer = AsyncServer(self) def __sigTERMhandler(self, signum, frame): log.debug("Caught signal %d. Exiting" % signum) self.quit() def quit(self): self.__asyncServer.stop() for c in self.__collectors: c.quit() if self.__buffer_thread is not None: self.__buffer_thread.quit() if self.__pusher_pid > -1: try: os.kill(self.__pusher_pid, signal.SIGTERM) except OSError as err: err = str(err) if err.find("No such process") > 0: log.info("Pusher process has gone.") self.__running = False def init(self): """Set up all necessary directories and etc""" if 'global' in config: global_vars = config['global'] set_global(global_vars) for n, cfg in config['output'].iteritems(): if n == 'buffer': if not 'directory' in cfg: print "ERROR: buffer directory not specified in config." return False buffer_dir = cfg['directory'] if os.path.exists(buffer_dir) and ( not os.path.isdir(buffer_dir)): print "ERROR: buffer directory exists but it is not a directory." return False if not os.path.exists(buffer_dir): log.info("Creating buffer directory %s.", buffer_dir) os.makedirs(buffer_dir) self.__outputs[n] = BufferOutput(cfg) elif n == 'kafka-http': self.__outputs[n] = KafkaHTTPOutput(cfg) elif n == 'aws': self.__outputs[n] = AWSOutput(cfg) elif n == 'file': self.__outputs[n] = FileOutput(cfg) elif 'class' in cfg: arguments = {} if 'arguments' in cfg: arguments = cfg['arguments'] self.__outputs[n] = init_object(cfg['class'], **arguments) if 'pusher' in config: if not 'directory' in config['pusher'] or not 'output' in config[ 'pusher']: print "ERROR: need to specify directory and output in pusher." return False if 'tailer' in config: self.__tailer = Tailer(config['tailer']) return True def run(self): # Install signal handlers signal.signal(signal.SIGTERM, self.__sigTERMhandler) signal.signal(signal.SIGINT, self.__sigTERMhandler) # Ensure unhandled exceptions are logged sys.excepthook = excepthook log.info("Reporting producer started at %s", datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")) if 'pusher' in config: pusher = Pusher(self.__outputs[config['pusher']['output']], config['pusher']['directory'], config['pusher'].get('batch', 1), config['pusher'].get('stats_on', False), config['pusher'].get('back_off_indicator', None)) pusher.daemon = True pusher.start() self.__pusher_pid = pusher.pid log.info("started pusher pid=%d", pusher.pid) if 'buffer' in self.__outputs: self.__buffer_thread = BufferThread(self.__outputs['buffer']) if 'collector' in config: for collector_config in config['collector']: log.debug("Initiating collector %s", collector_config) log.debug("Output object: %s", self.__outputs) c = Collector( collector_config, config['collector'][collector_config], self.__outputs[config['collector'][collector_config] ['output']], self.__tailer) self.__collectors.append(c) for c in self.__collectors: c.start() if self.__buffer_thread is not None: log.info("starting buffer thread") self.__buffer_thread.start() # Start the communication log.debug("Starting console communication") try: self.__asyncServer.start(self.__socket_file) except AsyncServerException as e: log.exception("Could not start socket server: %s", e) if 'pusher' in config: pusher.join() if self.__buffer_thread is not None and self.__buffer_thread.is_alive( ): self.__buffer_thread.join() for c in self.__collectors: if c.is_alive(): c.join() log.info("Reporting producer stopped at %s", datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")) def console(self, message): if message == "config": return json.dumps({"producer-config": self.config}) elif message == "show": return json.dumps( {"collectors": [c.info() for c in self.__collectors]}) elif message == "check": result = [] if 'buffer' in self.__outputs: if not self.__outputs['buffer'].check_dir.is_ok(): result.append("Buffer directory %s is full." % self.__outputs['buffer'].directory) for c in self.__collectors: info = c.info() if not info['is_running']: result.append("Collector %s has stopped." % info['name']) return json.dumps({"check-result": result}) elif message == "help": return "{\"system\": \"Command list: config, show, check\"}" else: return "{\"system\": \"Unknown command. Please run help to get a list of supported commands.\"}"
class ProducerDaemon(Daemon): def __init__(self, pidfile, socketfile, config, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): Daemon.__init__(self, pidfile, stdin, stdout, stderr) self.__running=True self.__socket_file=socketfile self.config=config self.__outputs={} self.__pusher_pid=-1 self.__tailer = None self.__buffer_thread=None self.__collectors=[] self.__asyncServer = AsyncServer(self) def __sigTERMhandler(self, signum, frame): log.debug("Caught signal %d. Exiting" % signum) self.quit() def quit(self): self.__asyncServer.stop() for c in self.__collectors: c.quit() if self.__buffer_thread is not None: self.__buffer_thread.quit() if self.__pusher_pid>-1: try: os.kill(self.__pusher_pid, signal.SIGTERM) except OSError as err: err = str(err) if err.find("No such process") > 0: log.info("Pusher process has gone.") self.__running=False def init(self): """Set up all necessary directories and etc""" if 'global' in config: global_vars = config['global'] set_global(global_vars) for n, cfg in config['output'].iteritems(): if n == 'buffer': if not 'directory' in cfg: print "ERROR: buffer directory not specified in config." return False buffer_dir = cfg['directory'] if os.path.exists(buffer_dir) and (not os.path.isdir(buffer_dir)): print "ERROR: buffer directory exists but it is not a directory." return False if not os.path.exists(buffer_dir): log.info("Creating buffer directory %s.", buffer_dir) os.makedirs(buffer_dir) self.__outputs[n] = BufferOutput(cfg) elif n == 'kafka-http': self.__outputs[n] = KafkaHTTPOutput(cfg) elif n == 'file': self.__outputs[n] = FileOutput(cfg) elif 'class' in cfg: arguments = {} if 'arguments' in cfg: arguments = cfg['arguments'] self.__outputs[n] = init_object(cfg['class'], **arguments) if 'pusher' in config: if not 'directory' in config['pusher'] or not 'output' in config['pusher']: print "ERROR: need to specify directory and output in pusher." return False if 'tailer' in config: self.__tailer = Tailer(config['tailer']) return True def run(self): # Install signal handlers signal.signal(signal.SIGTERM, self.__sigTERMhandler) signal.signal(signal.SIGINT, self.__sigTERMhandler) # Ensure unhandled exceptions are logged sys.excepthook = excepthook log.info("Reporting producer started at %s", datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")) if 'pusher' in config: pusher = Pusher(self.__outputs[config['pusher']['output']], config['pusher']['directory'], config['pusher'].get('batch', 1), config['pusher'].get('stats_on', False), config['pusher'].get('back_off_indicator', None)) pusher.daemon = True pusher.start() self.__pusher_pid = pusher.pid log.info("started pusher pid=%d", pusher.pid) if 'buffer' in self.__outputs: self.__buffer_thread = BufferThread(self.__outputs['buffer']) if 'collector' in config: for collector_config in config['collector']: log.debug("Initiating collector %s", collector_config) log.debug("self.__outputs: %s", self.__outputs) c = Collector(collector_config, config['collector'][collector_config], self.__outputs[config['collector'][collector_config]['output']], self.__tailer) self.__collectors.append(c) for c in self.__collectors: c.start() if self.__buffer_thread is not None: log.info("starting buffer thread") self.__buffer_thread.start() # Start the communication log.debug("Starting console communication") try: self.__asyncServer.start(self.__socket_file) except AsyncServerException as e: log.exception("Could not start socket server: %s", e) if 'pusher' in config: pusher.join() if self.__buffer_thread is not None and self.__buffer_thread.is_alive(): self.__buffer_thread.join() for c in self.__collectors: if c.is_alive(): c.join() log.info("Reporting producer stopped at %s", datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")) def console(self, message): if message=="config": return json.dumps({"producer-config":self.config}) elif message=="show": return json.dumps({"collectors": [c.info() for c in self.__collectors]}) elif message=="check": result=[] if 'buffer' in self.__outputs: if not self.__outputs['buffer'].check_dir.is_ok(): result.append("Buffer directory %s is full." % self.__outputs['buffer'].directory) for c in self.__collectors: info=c.info() if not info['is_running']: result.append("Collector %s has stopped." % info['name']) return json.dumps({"check-result":result}) elif message=="help": return "{\"system\": \"Command list: config, show, check\"}" else: return "{\"system\": \"Unknown command. Please run help to get a list of supported commands.\"}"