def init_logger(sd_loglevel=logging.WARN, stream_loglevel=logging.CRITICAL):
logging.setLogRecordFactory(LogRecordWithHexThereadID)
logger = logging.getLogger('deepfx')
logger.setLevel(sd_loglevel)
formatter = logging.Formatter('[%(hex_threadid)s] %(message)s')
if sd_loglevel:
import google
from google.cloud.logging import Client
from google.cloud.logging.handlers import CloudLoggingHandler
client = google.cloud.logging.Client.from_service_account_json(
os.environ.get('GOOGLE_SERVICE_ACCOUNT_JSON_PATH'))
handler = CloudLoggingHandler(client, name='deepfx')
handler.setLevel(sd_loglevel)
handler.setFormatter(formatter)
logger.addHandler(handler)
handler = None
if stream_loglevel:
handler = StreamHandler()
handler.setLevel(stream_loglevel)
handler.setFormatter(formatter)
logger.addHandler(handler)
handler = None
return logger
def setup_logger() -> object:
'''Setup main application logger
'''
# Setup uncaught exception handler
default_exception_handler()
if app_config['app_log_factory'] == 'gcp' and os.getenv(
'GOOGLE_APPLICATION_CREDENTIALS') is None:
app_config[
'app_log_factory'] = 'file' # Prevent set up GCP logging without credentials ENV variable
logging.error(
'GOOGLE_APPLICATION_CREDENTIALS ENV variable is missing, logging set to file output'
)
if app_config['app_log_factory'] == 'gcp':
# Disable whole cherrypy console logging
cherrypy.log.screen = False
logging.getLogger("cherrypy").propagate = False
# Connect GCP logging to default Python logger
logger = logging.getLogger()
# Remove original log handlers
for handler in logger.handlers:
logger.removeHandler(handler)
# Setup Google Cloud Logging
client = google.cloud.logging.Client()
# Setup CloudLoggingHandler(logging.StreamHandler) handler explicitly with Custom GCP Formatter
handler = CloudLoggingHandler(client,
labels={
'app_name': app_config['app_name'],
'app_version':
app_config['app_version'],
'app_environment':
app_config['app_env']
})
handler.setFormatter(CustomGCPFormatter())
# Setup Python logger explicitly with custom handler
setup_logging(handler)
elif app_config['app_log_factory'] == 'file':
# Load log configuration
logging.config.dictConfig(LOG_CONFIG)
# Custom app logger
logger = logging.getLogger('app')
else:
# Load log configuration
logging.config.dictConfig(LOG_CONFIG)
# Custom app logger
logger = logging.getLogger()
return logger
def add_cloud_logging_handler(logger: Logger):
client = google.cloud.logging.Client()
handler = CloudLoggingHandler(
client=client,
name="clouddq",
labels={
"name": APP_NAME,
"releaseId": APP_VERSION,
},
)
handler.setFormatter(JSONFormatter())
logger.addHandler(handler)
def get_handler(logName):
kwargs = {}
try:
kwargs['resource'] = get_log_resource_for_gce_instance()
except:
# Probably not on GCE ;-)
pass
# TODO: When we launched celery workers using prefork (multiprocessing: separate process per worker)
# we found that from google.cloud.logging.handlers.transports.background_thread.BackgroundThreadTransport
# stopped transmitting logs to GCP. We're not sure why, but as a workaround we switched to using
# a SyncTransport sub-class.
handler = CloudLoggingHandler(Client(), logName, transport=StructlogTransport, **kwargs)
handler.setFormatter(jsonlogger.JsonFormatter())
return handler
def init_gcloud_log(project_id, logger_name):
log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
if IS_DEV_ENV or IS_TEST_ENV:
logging.basicConfig(filename=logger_name + u'.log', level=logging.DEBUG, format=log_format)
else:
# see: https://googlecloudplatform.github.io/google-cloud-python/latest/logging-usage.html#cloud-logging-handler
# and https://github.com/GoogleCloudPlatform/getting-started-python/blob/master/6-pubsub/bookshelf/__init__.py#L40
client = google.cloud.logging.Client(project_id)
# NB: we should use AppEngineHandler for server at next google.cloud API update
# https://googlecloudplatform.github.io/google-cloud-python/latest/logging-handlers-app-engine.html
handler = CloudLoggingHandler(client, logger_name)
handler.setFormatter(logging.Formatter(log_format))
setup_logging(handler)
logging.getLogger().setLevel(logging.INFO)
logging.getLogger("readability.readability").setLevel(logging.WARNING) # very verbose package
def configure_gcp_logger(self):
# Imports the Google Cloud client library
if DiffgramLogger.logging_initialized.get(self.logger_name):
return logging.getLogger(self.logger_name)
from google.cloud import logging as gcp_logging
from google.cloud.logging.handlers import CloudLoggingHandler
# Instantiates a client
fmt_str = '[%(asctime)s] %(levelname)s %(module)s.py @ line %(lineno)d: %(message)s'
logging.basicConfig(level=logging.INFO, format=fmt_str)
logging_client = gcp_logging.Client()
handler = CloudLoggingHandler(logging_client, name=self.logger_name)
handler.setFormatter(logging.Formatter(fmt_str))
logger = logging.getLogger(self.logger_name)
logger.addHandler(handler)
logger.info('Logger {} setup success.'.format(self.logger_name))
DiffgramLogger.logging_initialized[self.logger_name] = True
return logger
def getLogger(name, level=__loglevel):
formatter = logging.Formatter('%(nasip)s %(nasid)s %(siteid)s %(msgtype)s %(message)s', '%a, %d %b %Y %H:%M:%S', )
logger = logging.getLogger(name)
logger.setLevel(level)
logger.addFilter(log_ctx)
if GRAYLOG_SERVER:
client = google.cloud.logging.Client()
cloud_handler = CloudLoggingHandler(client, name="radiusd")
logger.addHandler(cloud_handler)
setup_logging(cloud_handler)
cloud_handler.addFilter(log_ctx)
cloud_handler.setFormatter(formatter)
else:
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
return logger
import asyncio
import logging
import google.cloud.logging
from google.cloud.logging.handlers import CloudLoggingHandler
from discord import File
from discord.utils import find
from discord.ext import commands
from discord.ext.commands import when_mentioned_or
from datetime import datetime
logging_client = google.cloud.logging.Client()
cloud_logger = logging_client.logger('covid-19')
logger = logging.getLogger('covid-19')
logger.setLevel(logging.DEBUG)
handler = CloudLoggingHandler(logging_client)
handler.setFormatter(
logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)
class Coronavirus(commands.AutoShardedBot):
def __init__(self):
super().__init__(
command_prefix=when_mentioned_or(".c "),
activity=discord.Game(name="Loading..."),
)
self.remove_command('help')
self.load()
def load(self):
for filename in os.listdir('./cogs'):
try:
logger = logging.getLogger(__name__)
logger.setLevel('DEBUG')
fmframework_logger = logging.getLogger('fmframework')
if os.environ.get('DEPLOY_DESTINATION', None) == 'PROD':
import google.cloud.logging
from google.cloud.logging.handlers import CloudLoggingHandler, setup_logging
log_format = '%(funcName)s - %(message)s'
formatter = logging.Formatter(log_format)
client = google.cloud.logging.Client()
handler = CloudLoggingHandler(client, name="sarsooxyz")
handler.setFormatter(formatter)
logger.addHandler(handler)
fmframework_logger.addHandler(handler)
else:
log_format = '%(levelname)s %(name)s:%(funcName)s - %(message)s'
formatter = logging.Formatter(log_format)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
fmframework_logger.addHandler(stream_handler)
def get_handler(logName):
handler = CloudLoggingHandler(Client(), logName)
handler.setFormatter(jsonlogger.JsonFormatter())
return handler