def add_http_logstash_handler(self, host='http://logapi.misa.com.vn', port=80, app_id='test-log', level=logging.ERROR, database_path=None, headers=None): from logstash_async.handler import AsynchronousLogstashHandler from logstash_async.formatter import LogstashFormatter class HttpTransport: def __init__(self, host, port, **kwargs): self._host = host self._port = port self._headers = kwargs['headers'] if 'headers' in kwargs and kwargs['headers'] else {'Authorization': 'Basic bWlzYTpNaXNhQDIwMTk=', 'Content-Type': 'application/json; charset=utf-8'} def close(self): pass def send(self, data: dict, use_logging=None): for v in data: requests.post(url=self._host, data=v, headers=self._headers) class CustomFormatter(LogstashFormatter): def _move_extra_record_fields_to_prefix(self, message): super()._move_extra_record_fields_to_prefix(message) message['app_id'] = app_id handler = AsynchronousLogstashHandler(host, port, transport=HttpTransport(host, port, headers=headers), database_path=database_path or f'/tmp/logstash_{app_id}.db') handler.formatter = CustomFormatter(tags=[app_id]) handler.setLevel(level) self.handlers.append(handler) return self
def configure_logging(self): # Filters str_format_filter = dist_zero.logging.StrFormatFilter() context = { 'env': settings.DIST_ZERO_ENV, 'mode': runners.MODE_SIMULATED, 'runner': True, 'simulator_id': self.id, 'start_at': self._start_datetime, } if settings.LOGZ_IO_TOKEN: context['token'] = settings.LOGZ_IO_TOKEN context_filter = dist_zero.logging.ContextFilter(context) # Formatters human_formatter = dist_zero.logging.HUMAN_FORMATTER json_formatter = dist_zero.logging.JsonFormatter( '(asctime) (levelname) (name) (message)') # Handlers stdout_handler = logging.StreamHandler(sys.stdout) human_file_handler = logging.FileHandler('./.tmp/simulator.log') json_file_handler = logging.FileHandler('./.tmp/simulator.json.log') logstash_handler = AsynchronousLogstashHandler( settings.LOGSTASH_HOST, settings.LOGSTASH_PORT, database_path='./.tmp/logstash.db', ) stdout_handler.setLevel(logging.ERROR) human_file_handler.setLevel(logging.DEBUG) json_file_handler.setLevel(logging.DEBUG) logstash_handler.setLevel(logging.DEBUG) stdout_handler.setFormatter(human_formatter) human_file_handler.setFormatter(human_formatter) json_file_handler.setFormatter(json_formatter) logstash_handler.setFormatter(json_formatter) stdout_handler.addFilter(str_format_filter) human_file_handler.addFilter(str_format_filter) json_file_handler.addFilter(str_format_filter) json_file_handler.addFilter(context_filter) logstash_handler.addFilter(str_format_filter) logstash_handler.addFilter(context_filter) # Loggers dist_zero_logger = logging.getLogger('dist_zero') root_logger = logging.getLogger() dist_zero.logging.set_handlers(root_logger, [ json_file_handler, human_file_handler, logstash_handler, stdout_handler, ])
def add_logstash_handler(self, host, port, app_id, level=logging.ERROR, database_path=None): from logstash_async.handler import AsynchronousLogstashHandler from logstash_async.formatter import LogstashFormatter class CustomFormatter(LogstashFormatter): def _move_extra_record_fields_to_prefix(self, message): super()._move_extra_record_fields_to_prefix(message) message['app_id'] = app_id handler = AsynchronousLogstashHandler(host, port, database_path=database_path or f'/tmp/logstash_{app_id}.db') handler.formatter = CustomFormatter(tags=[app_id]) handler.setLevel(level) self.handlers.append(handler) return self
try: compute(" AND Lot:"+str(lot),str(lot)) except Exception as e: logger.error("Unable to compute Lot "+str(lot),exc_info=True) logger.info("<== "*10) logging.basicConfig(level=logging.INFO,format='%(asctime)s %(levelname)s %(module)s - %(funcName)s: %(message)s', datefmt="%Y-%m-%d %H:%M:%S") logger = logging.getLogger() lshandler=None if os.environ["USE_LOGSTASH"]=="true": logger.info ("Adding logstash appender") lshandler=AsynchronousLogstashHandler("logstash", 5001, database_path='logstash_test.db') lshandler.setLevel(logging.ERROR) logger.addHandler(lshandler) handler = TimedRotatingFileHandler("logs/"+MODULE+".log", when="d", interval=1, backupCount=30) logFormatter = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s') handler.setFormatter( logFormatter ) logger.addHandler(handler) logger.info("==============================") logger.info("Starting: %s" % MODULE) logger.info("Module: %s" %(VERSION)) logger.info("==============================")
def configure_logging(self): ''' Configure logging for a `MachineController` ''' # Filters str_format_filter = dist_zero.logging.StrFormatFilter() context = { 'env': settings.DIST_ZERO_ENV, 'mode': self.mode, 'runner': False, 'machine_id': self.id, 'machine_name': self.name, } if settings.LOGZ_IO_TOKEN: context['token'] = settings.LOGZ_IO_TOKEN context_filter = dist_zero.logging.ContextFilter(context) # Formatters human_formatter = dist_zero.logging.HUMAN_FORMATTER json_formatter = dist_zero.logging.JsonFormatter( '(asctime) (levelname) (name) (message)') # Handlers stdout_handler = logging.StreamHandler(sys.stdout) human_file_handler = logging.FileHandler( os.path.join(docker.DockerSimulatedHardware.CONTAINER_LOGS_DIR, 'output.log')) json_file_handler = logging.FileHandler( os.path.join(docker.DockerSimulatedHardware.CONTAINER_LOGS_DIR, 'output.json.log')) logstash_handler = AsynchronousLogstashHandler( settings.LOGSTASH_HOST, settings.LOGSTASH_PORT, database_path='/.logstash.db', ) stdout_handler.setLevel(logging.ERROR) human_file_handler.setLevel(logging.DEBUG) json_file_handler.setLevel(logging.DEBUG) logstash_handler.setLevel(logging.DEBUG) stdout_handler.setFormatter(human_formatter) human_file_handler.setFormatter(human_formatter) json_file_handler.setFormatter(json_formatter) logstash_handler.setFormatter(json_formatter) stdout_handler.addFilter(str_format_filter) human_file_handler.addFilter(str_format_filter) json_file_handler.addFilter(str_format_filter) json_file_handler.addFilter(context_filter) logstash_handler.addFilter(str_format_filter) logstash_handler.addFilter(context_filter) # Loggers dist_zero_logger = logging.getLogger('dist_zero') root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) dist_zero.logging.set_handlers(root_logger, [ json_file_handler, human_file_handler, logstash_handler, stdout_handler, ])