def get_logger(): logging.setLoggerClass(PyLogrus) logger = logging.getLogger(__name__) # type: PyLogrus logger.setLevel(logging.DEBUG) enabled_fields = [('name', 'logger_name'), ('asctime', 'service_timestamp'), ('levelname', 'level'), ('threadName', 'thread_name'), 'message', ('exception', 'exception_class'), ('stacktrace', 'stack_trace'), 'module', ('funcName', 'function')] formatter = JsonFormatter(datefmt='Z', enabled_fields=enabled_fields, indent=2, sort_keys=True) formatter.override_level_names({'CRITICAL': 'FATAL', 'WARNING': 'WARN'}) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) logger.addHandler(ch) return logger
def test_level_names_overriding(self): formatter = JsonFormatter() formatter.override_level_names({'CRITICAL': 'FATAL'}) log = self.get_logger(formatter) log.critical("test message") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertEqual(content['levelname'], 'FATAL')
def test_unicode(self): formatter = JsonFormatter() log = self.get_logger(formatter) log.debug("π π π π π") with open(self.filename, 'rb') as f: content = json.loads(f.readlines()[-1]) self.assertIn("\U0001f604 \U0001f601 \U0001f606 \U0001f605 \U0001f602", repr(content['message']))
def get_logger(): # POC is using a python port of the Golang logging library the final # version will use (when coded in golang). It's a bit un-pythonic but will # ease the move from PoC to final code. logging.setLoggerClass(PyLogrus) logger = logging.getLogger(__name__) # type: PyLogrus logger.setLevel(logging.DEBUG) enabled_fields = [('name', 'logger_name'), ('asctime', 'service_timestamp'), ('levelname', 'level'), ('threadName', 'thread_name'), 'message', ('exception', 'exception_class'), ('stacktrace', 'stack_trace'), 'module', ('funcName', 'function')] formatter = JsonFormatter(datefmt='Z', enabled_fields=enabled_fields, sort_keys=True) # Uncomment below to output structured logs in commandline-friendly text format #formatter = TextFormatter(datefmt='Z', colorize=False) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) logger.addHandler(ch) return logger
def test_extra_fields(self): formatter = JsonFormatter() log = self.get_logger(formatter) log.withFields({'user': '******'}).info("test message") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertIn('user', content) self.assertEqual(content['user'], 'John Doe')
def __init__(self, config_path): self.config = self.get_config(config_path) fh = logging.FileHandler( "logs/sitealgo" + str(self.config["site_id"]) + ".log", 'w+') fh.setLevel(logging.DEBUG) jsonformatter = JsonFormatter(datefmt='Z') fh.setFormatter(jsonformatter) logger.addHandler(fh) log = logger.withFields({"node": "site-algo"})
def test_date_format(self): formatter = JsonFormatter(datefmt='%m/%d/%Y %I:%M:%S %p') log = self.get_logger(formatter) log.info("test a date in custom format") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) pattern = "^\d{2}\/\d{2}\/\d{4} \d{2}:\d{2}:\d{2} [AP]M$" if sys.version_info >= (3, 1): # Python version >= 3.1 self.assertRegex(content['asctime'], pattern) else: self.assertRegexpMatches(content['asctime'], pattern)
def test_date_format_zulu(self): formatter = JsonFormatter(datefmt='Z') log = self.get_logger(formatter) log.info("test a date in Zulu format") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) pattern = "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$" if sys.version_info >= (3, 1): # Python version >= 3.1 self.assertRegex(content['asctime'], pattern) else: self.assertRegexpMatches(content['asctime'], pattern)
def test_message_with_prefix(self): formatter = JsonFormatter() log = self.get_logger(formatter) log_pfx = log.withPrefix("[API]") log_pfx.info("Log message with the prefix") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertEqual(content['message'], "[API] Log message with the prefix") log_pfx.critical("Another one log message with the prefix") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertEqual(content['message'], "[API] Another one log message with the prefix")
def test_json_encoder(self): class MyEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, datetime.datetime): return obj.isoformat() else: return super().default(obj) formatter = JsonFormatter(json_encoder=MyEncoder) log = self.get_logger(formatter, name="json-encoder") now = datetime.datetime.now() log.withFields({"some_datetime": now}).debug("TEST") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertIn("some_datetime", content) self.assertEqual(content["some_datetime"], now.isoformat()) self.assertIn("message", content) self.assertEqual(content["message"], "TEST")
def test_levels(self): formatter = JsonFormatter() log = self.get_logger(formatter) log.debug("test_debug_output") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) # print(content) self.assertIn('levelname', content) self.assertEqual(content['levelname'], 'DEBUG') self.assertIn('message', content) self.assertEqual(content['message'], 'test_debug_output') log.info("test_info_output") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertIn('levelname', content) self.assertEqual(content['levelname'], 'INFO') self.assertIn('message', content) self.assertEqual(content['message'], 'test_info_output') log.warning("test_warning_output") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertIn('levelname', content) self.assertEqual(content['levelname'], 'WARNING') self.assertIn('message', content) self.assertEqual(content['message'], 'test_warning_output') log.error("test_error_output") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertIn('levelname', content) self.assertEqual(content['levelname'], 'ERROR') self.assertIn('message', content) self.assertEqual(content['message'], 'test_error_output') log.critical("test_critical_output") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertIn('levelname', content) self.assertEqual(content['levelname'], 'CRITICAL') self.assertIn('message', content) self.assertEqual(content['message'], 'test_critical_output')
def test_contextual_logging(self): formatter = JsonFormatter() log = self.get_logger(formatter) log_ctx = log.withFields({'context': 1}) log_ctx.withFields({'user': '******'}).info("contextual logger") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertIn('context', content) self.assertEqual(content['context'], 1) self.assertIn('user', content) self.assertEqual(content['user'], 'John Doe') log.withFields({'company': 'Awesome Company'}).info("default logger") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertNotIn('context', content) self.assertIn('company', content) self.assertEqual(content['company'], 'Awesome Company')
def test_enabled_fields(self): enabled_fields = [('asctime', 'service_timestamp'), ('levelname', 'level'), ('threadName', 'thread_name'), 'message', ('exception', 'exception_class'), ('stacktrace', 'stack_trace'), ('funcName', 'function')] formatter = JsonFormatter(enabled_fields=enabled_fields) log = self.get_logger(formatter) log.info("test message") with open(self.filename) as f: content = json.loads(f.readlines()[-1]) self.assertEqual(len(content), 7) self.assertIn('service_timestamp', content) self.assertIn('level', content) self.assertIn('thread_name', content) self.assertIn('message', content) self.assertIn('exception_class', content) self.assertIn('stack_trace', content) self.assertIn('function', content)
def __init__(self, config_path): self.config = self.get_config(config_path) # Setup logging tool logging.setLoggerClass(PyLogrus) logger = logging.getLogger(__name__) # type: PyLogrus logger.setLevel(logging.DEBUG) formatter = TextFormatter(datefmt='Z', colorize=True) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) jsonformatter = JsonFormatter(datefmt='Z') fh = logging.FileHandler("logs/cloudalgo.log", 'w+') fh.setLevel(logging.DEBUG) fh.setFormatter(jsonformatter) logger.addHandler(ch) logger.addHandler(fh) self.log = logger.withFields({"node": "cloud-algo"})
logger.warning("WARNING MESSAGE") logger.error("ERROR MESSAGE") print("γ‘γγ»γΌγΈγ«Prefixγγ€γγ") logger = logger.withPrefix("[API]") logger.debug("DEBUG MESSAGE") print("γ‘γγ»γΌγΈγ«γγ£γΌγ«γγθΏ½ε γγ") logger.withFields({'error_code': 404}).info("INFO MESSAGE") print("JSONε½’εΌγ§εΊε") logger = logging.getLogger(__name__) enabled_fields = [('name', 'logger_name'), ('asctime', 'service_timestamp'), ('levelname', 'level'), ('threadName', 'thread_name'), 'message', ('exception', 'exception_class'), ('stacktrace', 'stack_trace'), 'module', ('funcName', 'function')] formatter = JsonFormatter(datefmt='Z', enabled_fields=enabled_fields, indent=2, sort_keys=True) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) logger.addHandler(ch) logger.debug("DEBUG MESSAGE") logger.info("INFO MESSAGE") logger.warning("WARNING MESSAGE") logger.error("ERROR MESSAGE")
if __name__ == "__main__": max_iters = 100 iters_per_epoch = 10 learning_rate = 1e-4 batch_size = 16 logging.setLoggerClass(PyLogrus) logger = logging.getLogger(__name__) # type: PyLogrus logger.setLevel(logging.DEBUG) formatter = TextFormatter(datefmt='Z', colorize=True) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) jsonformatter = JsonFormatter(datefmt='Z') fh = logging.FileHandler("logs/resnet_baseline.log", 'w+') fh.setLevel(logging.DEBUG) fh.setFormatter(jsonformatter) logger.addHandler(ch) logger.addHandler(fh) log = logger.withFields({"node": "baseline"}) model = torchvision.models.resnet18(pretrained=True) in_features = model.fc.in_features model.fc = torch.nn.Linear(in_features, 2) optimizer = torch.optim.SGD(model.parameters(), learning_rate) criterion = torch.nn.CrossEntropyLoss()