def add_http_logstash_handler(self, host='http://logapi.misa.com.vn', port=80, app_id='test-log', level=logging.ERROR, database_path=None,
headers=None):
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
class HttpTransport:
def __init__(self, host, port, **kwargs):
self._host = host
self._port = port
self._headers = kwargs['headers'] if 'headers' in kwargs and kwargs['headers'] else {'Authorization': 'Basic bWlzYTpNaXNhQDIwMTk=',
'Content-Type': 'application/json; charset=utf-8'}
def close(self):
pass
def send(self, data: dict, use_logging=None):
for v in data:
requests.post(url=self._host, data=v, headers=self._headers)
class CustomFormatter(LogstashFormatter):
def _move_extra_record_fields_to_prefix(self, message):
super()._move_extra_record_fields_to_prefix(message)
message['app_id'] = app_id
handler = AsynchronousLogstashHandler(host, port, transport=HttpTransport(host, port, headers=headers),
database_path=database_path or f'/tmp/logstash_{app_id}.db')
handler.formatter = CustomFormatter(tags=[app_id])
handler.setLevel(level)
self.handlers.append(handler)
return self
def configure_logging(self):
# Filters
str_format_filter = dist_zero.logging.StrFormatFilter()
context = {
'env': settings.DIST_ZERO_ENV,
'mode': runners.MODE_SIMULATED,
'runner': True,
'simulator_id': self.id,
'start_at': self._start_datetime,
}
if settings.LOGZ_IO_TOKEN:
context['token'] = settings.LOGZ_IO_TOKEN
context_filter = dist_zero.logging.ContextFilter(context)
# Formatters
human_formatter = dist_zero.logging.HUMAN_FORMATTER
json_formatter = dist_zero.logging.JsonFormatter(
'(asctime) (levelname) (name) (message)')
# Handlers
stdout_handler = logging.StreamHandler(sys.stdout)
human_file_handler = logging.FileHandler('./.tmp/simulator.log')
json_file_handler = logging.FileHandler('./.tmp/simulator.json.log')
logstash_handler = AsynchronousLogstashHandler(
settings.LOGSTASH_HOST,
settings.LOGSTASH_PORT,
database_path='./.tmp/logstash.db',
)
stdout_handler.setLevel(logging.ERROR)
human_file_handler.setLevel(logging.DEBUG)
json_file_handler.setLevel(logging.DEBUG)
logstash_handler.setLevel(logging.DEBUG)
stdout_handler.setFormatter(human_formatter)
human_file_handler.setFormatter(human_formatter)
json_file_handler.setFormatter(json_formatter)
logstash_handler.setFormatter(json_formatter)
stdout_handler.addFilter(str_format_filter)
human_file_handler.addFilter(str_format_filter)
json_file_handler.addFilter(str_format_filter)
json_file_handler.addFilter(context_filter)
logstash_handler.addFilter(str_format_filter)
logstash_handler.addFilter(context_filter)
# Loggers
dist_zero_logger = logging.getLogger('dist_zero')
root_logger = logging.getLogger()
dist_zero.logging.set_handlers(root_logger, [
json_file_handler,
human_file_handler,
logstash_handler,
stdout_handler,
])
def add_logstash_handler(self, host, port, app_id, level=logging.ERROR, database_path=None):
from logstash_async.handler import AsynchronousLogstashHandler
from logstash_async.formatter import LogstashFormatter
class CustomFormatter(LogstashFormatter):
def _move_extra_record_fields_to_prefix(self, message):
super()._move_extra_record_fields_to_prefix(message)
message['app_id'] = app_id
handler = AsynchronousLogstashHandler(host, port, database_path=database_path or f'/tmp/logstash_{app_id}.db')
handler.formatter = CustomFormatter(tags=[app_id])
handler.setLevel(level)
self.handlers.append(handler)
return self
try:
compute(" AND Lot:"+str(lot),str(lot))
except Exception as e:
logger.error("Unable to compute Lot "+str(lot),exc_info=True)
logger.info("<== "*10)
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(levelname)s %(module)s - %(funcName)s: %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
logger = logging.getLogger()
lshandler=None
if os.environ["USE_LOGSTASH"]=="true":
logger.info ("Adding logstash appender")
lshandler=AsynchronousLogstashHandler("logstash", 5001, database_path='logstash_test.db')
lshandler.setLevel(logging.ERROR)
logger.addHandler(lshandler)
handler = TimedRotatingFileHandler("logs/"+MODULE+".log",
when="d",
interval=1,
backupCount=30)
logFormatter = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s')
handler.setFormatter( logFormatter )
logger.addHandler(handler)
logger.info("==============================")
logger.info("Starting: %s" % MODULE)
logger.info("Module: %s" %(VERSION))
logger.info("==============================")
def configure_logging(self):
'''
Configure logging for a `MachineController`
'''
# Filters
str_format_filter = dist_zero.logging.StrFormatFilter()
context = {
'env': settings.DIST_ZERO_ENV,
'mode': self.mode,
'runner': False,
'machine_id': self.id,
'machine_name': self.name,
}
if settings.LOGZ_IO_TOKEN:
context['token'] = settings.LOGZ_IO_TOKEN
context_filter = dist_zero.logging.ContextFilter(context)
# Formatters
human_formatter = dist_zero.logging.HUMAN_FORMATTER
json_formatter = dist_zero.logging.JsonFormatter(
'(asctime) (levelname) (name) (message)')
# Handlers
stdout_handler = logging.StreamHandler(sys.stdout)
human_file_handler = logging.FileHandler(
os.path.join(docker.DockerSimulatedHardware.CONTAINER_LOGS_DIR,
'output.log'))
json_file_handler = logging.FileHandler(
os.path.join(docker.DockerSimulatedHardware.CONTAINER_LOGS_DIR,
'output.json.log'))
logstash_handler = AsynchronousLogstashHandler(
settings.LOGSTASH_HOST,
settings.LOGSTASH_PORT,
database_path='/.logstash.db',
)
stdout_handler.setLevel(logging.ERROR)
human_file_handler.setLevel(logging.DEBUG)
json_file_handler.setLevel(logging.DEBUG)
logstash_handler.setLevel(logging.DEBUG)
stdout_handler.setFormatter(human_formatter)
human_file_handler.setFormatter(human_formatter)
json_file_handler.setFormatter(json_formatter)
logstash_handler.setFormatter(json_formatter)
stdout_handler.addFilter(str_format_filter)
human_file_handler.addFilter(str_format_filter)
json_file_handler.addFilter(str_format_filter)
json_file_handler.addFilter(context_filter)
logstash_handler.addFilter(str_format_filter)
logstash_handler.addFilter(context_filter)
# Loggers
dist_zero_logger = logging.getLogger('dist_zero')
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
dist_zero.logging.set_handlers(root_logger, [
json_file_handler,
human_file_handler,
logstash_handler,
stdout_handler,
])
