def init_app(app):
# Output logs on stderr
fmt = '{record.channel}: {record.message}'
stderr = logbook.StderrHandler(format_string=fmt)
stderr.push_application()
def logbook_factory(*args, **kwargs):
# Logger given to structlog
level = app.debug and logbook.DEBUG or logbook.INFO
return logbook.Logger(level=level, *args, **kwargs)
# Setup structlog over logbook
processors = [
# structlog.stdlib.filter_by_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
# structlog.processors.format_exc_info,
]
# send to mozdef before formatting into a string
mozdef = app.config.get('MOZDEF_TARGET', None)
if mozdef:
processors.append(mozdef)
processors.append(UnstructuredRenderer())
structlog.configure(
context_class=structlog.threadlocal.wrap_dict(dict),
processors=processors,
logger_factory=logbook_factory,
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def setup_logging(config):
"""Setup structured logging, and emit `request.summary` event on each
request, as recommanded by Mozilla Services standard:
* https://mana.mozilla.org/wiki/display/CLOUDSERVICES/Logging+Standard
* http://12factor.net/logs
"""
settings = config.get_settings()
renderer_klass = config.maybe_dotted(settings['cliquet.logging_renderer'])
renderer = renderer_klass(settings)
structlog.configure(
# Share the logger context by thread.
context_class=structlog.threadlocal.wrap_dict(dict),
# Integrate with Pyramid logging facilities.
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
# Setup logger output format.
processors=[
structlog.stdlib.filter_by_level,
structlog.processors.format_exc_info,
renderer,
])
def on_new_request(event):
request = event.request
# Save the time the request was received by the server.
event.request._received_at = utils.msec_time()
# New logger context, with infos for request summary logger.
logger.new(agent=request.headers.get('User-Agent'),
path=event.request.path,
method=request.method,
querystring=dict(request.GET),
uid=request.authenticated_userid,
lang=request.headers.get('Accept-Language'),
auth_type=getattr(request, 'auth_type', None),
errno=None)
config.add_subscriber(on_new_request, NewRequest)
def on_new_response(event):
response = event.response
request = event.request
# Compute the request processing time in msec (-1 if unknown)
current = utils.msec_time()
duration = current - getattr(request, '_received_at', current - 1)
isotimestamp = datetime.fromtimestamp(current/1000).isoformat()
# Bind infos for request summary logger.
logger.bind(time=isotimestamp,
code=response.status_code,
t=duration)
# Ouput application request summary.
logger.info('request.summary')
config.add_subscriber(on_new_response, NewResponse)
def run():
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger('sdc.rabbit').setLevel(logging.INFO)
# These structlog settings allow bound fields to persist between classes
structlog.configure(logger_factory=LoggerFactory(), context_class=wrap_dict(dict))
logger = structlog.getLogger()
logger.info('Starting SDX Downstream', version=__version__)
message_processor = MessageProcessor()
quarantine_publisher = QueuePublisher(
urls=settings.RABBIT_URLS,
queue=settings.RABBIT_QUARANTINE_QUEUE
)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type='topic',
rabbit_queue=settings.RABBIT_QUEUE,
rabbit_urls=settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=message_processor.process
)
try:
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
def configure(config_string='', log_json=False):
# configure structlog
processors = [
log_listeners, # before level filtering
structlog.stdlib.filter_by_level,
structlog.processors.StackInfoRenderer()
]
if log_json:
processors.append(JSONRenderer(sort_keys=True))
else:
processors.extend([
structlog.processors.ExceptionPrettyPrinter(file=None),
KeyValueRenderer(sort_keys=True, key_order=None)
])
structlog.configure(
processors=processors,
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=BoundLoggerTrace,
# later calls on configure() dont have any effect on already cached loggers
cache_logger_on_first_use=True,
)
# configure standard logging
if log_json:
format = JSON_FORMAT
else:
format = PRINT_FORMAT
setup_stdlib_logging(DEFAULT_LOGLEVEL, format)
if config_string:
configure_loglevels(config_string)
def run():
args = ap.parse_args()
if args.ls:
print('\n'.join(parser.listPluginNames()))
sys.exit(0)
if args.verbose:
structlog.configure(logger_factory=structlog.PrintLoggerFactory(sys.stderr))
infile = BytesIO(sys.stdin.read())
try:
parsed = parse(infile, exclude=args.exclude)
except NoWillingParsers:
if args.strict:
raise
else:
infile.seek(0)
sys.stdout.write(infile.read())
sys.exit(0)
if args.format == 'yaml':
print(yaml.safe_dump(parsed, default_flow_style=False))
elif args.format == 'json':
print(json.dumps(parsed))
elif args.format == 'grep':
from ppo.output import giganticGrep
giganticGrep(parsed, sys.stdout)
def configure_logging(config):
structlog.configure(processors=processors,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True)
config.setdefault('logging', {})
config['logging'].setdefault('version', 1)
config['logging'].setdefault('handlers', {})
config['logging'].setdefault('formatters', {})
config['logging'].setdefault('loggers', {})
config['logging']['handlers'].setdefault('raw', {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'raw',
})
config['logging']['loggers'].setdefault('root', {
'handlers': ['raw'],
'level': 'DEBUG',
'propagate': False,
})
config['logging']['loggers'].setdefault('graphite_api', {
'handlers': ['raw'],
'level': 'DEBUG',
})
config['logging']['formatters']['raw'] = {'()': StructlogFormatter}
dictConfig(config['logging'])
if 'path' in config:
logger.info("loading configuration", path=config['path'])
else:
logger.info("loading default configuration")
def update_logging(instance_id, vcore_id):
"""
Add the vcore id to the structured logger
:param vcore_id: The assigned vcore id
:return: structure logger
"""
def add_exc_info_flag_for_exception(_, name, event_dict):
if name == 'exception':
event_dict['exc_info'] = True
return event_dict
def add_instance_id(_, __, event_dict):
event_dict['instance_id'] = instance_id
return event_dict
def add_vcore_id(_, __, event_dict):
event_dict['vcore_id'] = vcore_id
return event_dict
processors = [
add_exc_info_flag_for_exception,
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
add_instance_id,
add_vcore_id,
FluentRenderer(),
]
structlog.configure(processors=processors)
# Mark first line of log
log = structlog.get_logger()
log.info("updated-logger")
return log
def global_setup(config):
"""Perform global cofiguration. In a given process, this should only
ever be called with a single configuration instance. Doing otherwise
will result in a runtime exception.
"""
global _global_config
if _global_config is None:
_global_config = config
# this breaks with unicode :(
connection.setup([str(v) for v in config.CASSANDRA_CLUSTER],
consistency=config.CASSANDRA_CONSISTENCY)
processors = [
_capture_stack_trace,
_format_event,
]
if config.PRETTY_LOGGING:
processors.append(structlog.processors.ExceptionPrettyPrinter())
processors.append(structlog.processors.KeyValueRenderer())
else:
processors.append(structlog.processors.JSONRenderer())
structlog.configure(
processors=processors
)
elif _global_config != config:
raise Exception('global_setup called twice with different '
'configurations')
def configure_logging(app):
if app.config.get('JSON_STRUCTURED_LOGGING'):
processors = [
structlog.stdlib.filter_by_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
mozdef_format,
structlog.processors.JSONRenderer()
]
else:
processors = [
structlog.stdlib.filter_by_level,
structlog.stdlib.PositionalArgumentsFormatter(),
UnstructuredRenderer()
]
if app.config.get('JSON_STRUCTURED_LOGGING') and stdout_log:
# structlog has combined all of the interesting data into the
# (JSON-formatted) message, so only log that
stdout_log.setFormatter(logging.Formatter('%(message)s'))
structlog.configure(
context_class=structlog.threadlocal.wrap_dict(dict),
processors=processors,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def get_tiger():
"""
Sets up logging and returns a new tasktiger instance.
"""
structlog.configure(
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
)
logging.basicConfig(format='%(message)s')
conn = redis.Redis(db=TEST_DB)
tiger = TaskTiger(connection=conn, config={
# We need this 0 here so we don't pick up scheduled tasks when
# doing a single worker run.
'SELECT_TIMEOUT': 0,
'LOCK_RETRY': DELAY*2.,
'DEFAULT_RETRY_METHOD': fixed(DELAY, 2),
'BATCH_QUEUES': {
'batch': 3,
}
})
tiger.log.setLevel(logging.CRITICAL)
return tiger
def test_structlog_processor(self):
try:
# Use ReturnLogger for testing
structlog.configure(
processors=[tasktiger_processor],
context_class=dict,
logger_factory=structlog.ReturnLoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
# Run a simple task. Logging output is verified in
# the task.
self.tiger.delay(logging_task)
queues = self._ensure_queues(queued={"default": 1})
task = queues["queued"]["default"][0]
assert task["func"] == "tests.test_logging:logging_task"
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={"default": 0})
assert not self.conn.exists("t:task:%s" % task["id"])
finally:
structlog.configure(
processors=[
structlog.stdlib.add_log_level,
structlog.stdlib.filter_by_level,
structlog.processors.TimeStamper(fmt="iso", utc=True),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.JSONRenderer(),
],
context_class=dict,
logger_factory=structlog.ReturnLoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def configure_logging(logging_levels, plain=False):
_remove_all_existing_log_handlers()
renderer = (
PlainRenderer() if plain else
structlog.processors.JSONRenderer())
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt='iso'),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
renderer
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
handler = logging.StreamHandler(sys.stdout)
root_logger = logging.getLogger()
root_logger.addHandler(handler)
for logger, level in logging_levels.items():
if logger.lower() == 'root':
logger = ''
logging.getLogger(logger).setLevel(level.upper())
def run():
structlog.configure(
processors=[
structlog.processors.StackInfoRenderer(),
structlog.twisted.JSONRenderer()
],
context_class=dict,
logger_factory=structlog.twisted.LoggerFactory(),
wrapper_class=structlog.twisted.BoundLogger,
cache_logger_on_first_use=True,
)
# grab all of the events that are dispatched to stdlib logger
# new relic uses this.
handler = logging.StreamHandler(sys.stdout)
root_logger = logging.getLogger()
root_logger.addHandler(handler)
# start the twisted logger
twLog.startLogging(sys.stdout)
# api is the WSGI resource returned by Falcon.
api = falcon.API()
api.add_route('/quote', QuoteResource())
app = newrelic.agent.WSGIApplicationWrapper(api)
resource = WSGIResource(reactor, reactor.getThreadPool(), app)
site = Site(resource)
reactor.listenTCP(port=8713, factory=site)
reactor.run()
def init_logger(level):
logging.basicConfig()
formatter = logging.Formatter('%(message)s')
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
root_logger = logging.getLogger()
root_logger.removeHandler(root_logger.handlers[0])
root_logger.setLevel(level)
root_logger.addHandler(handler)
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt='iso'),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.KeyValueRenderer()
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def get_tiger():
"""
Sets up logging and returns a new tasktiger instance.
"""
structlog.configure(
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
)
logging.basicConfig(format='%(message)s')
conn = redis.Redis(db=TEST_DB, decode_responses=True)
tiger = TaskTiger(connection=conn, config={
# We need this 0 here so we don't pick up scheduled tasks when
# doing a single worker run.
'SELECT_TIMEOUT': 0,
'ACTIVE_TASK_UPDATE_TIMEOUT': 2 * DELAY,
'REQUEUE_EXPIRED_TASKS_INTERVAL': DELAY,
'LOCK_RETRY': DELAY * 2.,
'DEFAULT_RETRY_METHOD': fixed(DELAY, 2),
'BATCH_QUEUES': {
'batch': 3,
},
'SINGLE_WORKER_QUEUES': ['swq'],
})
tiger.log.setLevel(logging.CRITICAL)
return tiger
def init_logger(project_name,
channel=None,
level=logbook.INFO,
handler=None,
PAPERTRAIL_HOST=None,
PAPERTRAIL_PORT=None,
SENTRY_DSN=None,
MOZDEF=None,
flask_app=None,
timestamp=False,
):
if not channel:
channel = os.environ.get('APP_CHANNEL')
if channel and channel not in CHANNELS:
raise Exception('Initilizing logging with channel `{}`. It should be one of: {}'.format(channel, ', '.join(CHANNELS)))
# By default output logs on stderr
if handler is None:
fmt = '{record.channel}: {record.message}'
handler = logbook.StderrHandler(level=level, format_string=fmt)
handler.push_application()
# Log to papertrail
if channel and PAPERTRAIL_HOST and PAPERTRAIL_PORT:
setup_papertrail(project_name, channel, PAPERTRAIL_HOST, PAPERTRAIL_PORT)
# Log to sentry
if channel and SENTRY_DSN:
setup_sentry(project_name, channel, SENTRY_DSN, flask_app)
def logbook_factory(*args, **kwargs):
# Logger given to structlog
logbook.compat.redirect_logging()
return logbook.Logger(level=level, *args, **kwargs)
# Setup structlog over logbook
processors = [
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
]
if timestamp is True:
processors.append(structlog.processors.TimeStamper(fmt='%Y-%m-%d %H:%M:%S'))
# send to mozdef before formatting into a string
if channel and MOZDEF:
processors.append(setup_mozdef(project_name, channel, MOZDEF))
processors.append(UnstructuredRenderer())
structlog.configure(
context_class=structlog.threadlocal.wrap_dict(dict),
processors=processors,
logger_factory=logbook_factory,
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def configure_logging(log_format, utc, endpoint):
processors = [
TimeStamper(
key='@timestamp',
utc=utc,
),
]
if endpoint.startswith('file://'):
path = endpoint[7:]
if path == '/dev/stdout':
stream = sys.stdout
elif path == '/dev/stderr':
stream = sys.stderr
else:
stream = open(path, 'w')
logger_factory = structlog.PrintLoggerFactory(file=stream)
if log_format == 'kv':
processors.append(structlog.processors.KeyValueRenderer(
sort_keys=True,
key_order=['@timestamp', 'event'],
))
else:
processors.append(structlog.processors.JSONRenderer(
sort_keys=True,
))
elif endpoint.startswith('fluent://'):
utc = True
logger_factory = FluentLoggerFactory.from_url(endpoint)
else:
raise ValueError('Invalid logging endpoint "%s".' % endpoint)
structlog.configure(
processors=processors,
logger_factory=logger_factory,
)
def get_logger(level=logging.DEBUG, name=None, stream=DEFAULT_STREAM):
"""Configure and return a logger with structlog and stdlib."""
wrap_dict_class = structlog.threadlocal.wrap_dict(dict)
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_log_level,
structlog.stdlib.add_logger_name,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt='iso'),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.JSONRenderer(sort_keys=True)
],
context_class=wrap_dict_class,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True)
log = structlog.get_logger(name)
if not _has_streamhandler(logging.getLogger(name),
level=level, stream=stream):
streamhandler = logging.StreamHandler(stream)
streamhandler.setLevel(level)
streamhandler.setFormatter(logging.Formatter(fmt=LOG_FORMAT))
log.addHandler(streamhandler)
log.setLevel(level)
return log
def test_format_event_basic(self):
processors = [teeth_overlord.service._format_event,
_return_event_processor]
structlog.configure(processors=processors)
log = structlog.wrap_logger(structlog.ReturnLogger())
logged_msg = log.msg("hello {word}", word='world')
self.assertEqual(logged_msg, "hello world")
def configure_structlog():
"""
Make structlog comply with all of our options.
"""
from django.conf import settings
import logging
import structlog
from sentry import options
from sentry.logging import LoggingFormat
WrappedDictClass = structlog.threadlocal.wrap_dict(dict)
kwargs = {
'context_class': WrappedDictClass,
'wrapper_class': structlog.stdlib.BoundLogger,
'cache_logger_on_first_use': True,
'processors': [
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.format_exc_info,
structlog.processors.StackInfoRenderer(),
]
}
fmt_from_env = os.environ.get('SENTRY_LOG_FORMAT')
if fmt_from_env:
settings.SENTRY_OPTIONS['system.logging-format'] = fmt_from_env.lower()
fmt = options.get('system.logging-format')
if fmt == LoggingFormat.HUMAN:
from sentry.logging.handlers import HumanRenderer
kwargs['processors'].extend([
structlog.processors.ExceptionPrettyPrinter(),
HumanRenderer(),
])
elif fmt == LoggingFormat.MACHINE:
from sentry.logging.handlers import JSONRenderer
kwargs['processors'].append(JSONRenderer())
structlog.configure(**kwargs)
lvl = os.environ.get('SENTRY_LOG_LEVEL')
if lvl and lvl not in logging._levelNames:
raise AttributeError('%s is not a valid logging level.' % lvl)
settings.LOGGING['root'].update({
'level': lvl or settings.LOGGING['default_level']
})
if lvl:
for logger in settings.LOGGING['overridable']:
try:
settings.LOGGING['loggers'][logger].update({
'level': lvl
})
except KeyError:
raise KeyError('%s is not a defined logger.' % logger)
logging.config.dictConfig(settings.LOGGING)
def init_logging(log_dir, name):
if not os.path.exists(log_dir):
os.mkdir(log_dir)
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'plain': {
'()': ProcessorFormatter,
'processor': structlog.dev.ConsoleRenderer(), # TODO: ConsoleRenderer without coloring
},
'colored': {
'()': ProcessorFormatter,
'processor': structlog.dev.ConsoleRenderer(),
},
},
'handlers': {
'default': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'colored',
},
'file': {
'level': 'DEBUG',
'class': 'logging.handlers.WatchedFileHandler',
'filename': os.path.join(log_dir, '{}.log'.format(name)),
'formatter': 'plain',
},
},
'loggers': {
'': {
'handlers': ['default', 'file'],
'level': 'DEBUG',
'propagate': True,
},
'asyncio': {
'propagate': False,
},
'telnetlib3': {
'propagate': False,
},
}
})
structlog.configure(
processors=[
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt='%Y-%m-%d %H:%M:%S'),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
event_dict_to_message,
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
sys.excepthook = uncaught_exception
def test_no_format_keys(self):
"""Check that we get an exception if you don't provide enough keys to
format a log message requiring format
"""
processors = [teeth_overlord.service._format_event,
_return_event_processor]
structlog.configure(processors=processors)
log = structlog.wrap_logger(structlog.ReturnLogger())
self.assertRaises(KeyError, log.msg, "hello {word}")
def setUp(self):
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger('sdc.rabbit').setLevel(logging.INFO)
structlog.configure(logger_factory=LoggerFactory(), context_class=wrap_dict(dict))
self.message_processor = MessageProcessor()
def raise_on_logger_msg():
def proc(logger, method_name, event_dict):
if method_name in ('warning', 'error'):
if 'exc_info' in event_dict:
raise event_dict['exc_info']
if not event_dict['event'].startswith(('rate limited', 'no-result')):
raise RuntimeError(event_dict['event'])
return event_dict['event']
structlog.configure([proc])
def includeme(config):
structlog.configure(
processors=[
structlog.processors.KeyValueRenderer(
key_order=['event', 'request_id'],
),
],
context_class=structlog.threadlocal.wrap_dict(dict),
logger_factory=structlog.stdlib.LoggerFactory(),
)
def setUp(self):
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger('sdc.rabbit').setLevel(logging.INFO)
structlog.configure(logger_factory=LoggerFactory(), context_class=wrap_dict(dict))
survey = json.loads(cora_survey)
self.processor = CoraProcessor(survey, ftpconn)
self.processor.ftp.unzip_and_deliver = MagicMock(return_value=True)
def setup_logging(log_config, instance_id, verbosity_adjust=0, fluentd=None):
"""
Set up logging such that:
- The primary logging entry method is structlog
(see http://structlog.readthedocs.io/en/stable/index.html)
- By default, the logging backend is Python standard lib logger
- Alternatively, fluentd can be configured with to be the backend,
providing direct bridge to a fluent logging agent.
"""
def add_exc_info_flag_for_exception(_, name, event_dict):
if name == 'exception':
event_dict['exc_info'] = True
return event_dict
def add_instance_id(_, __, event_dict):
event_dict['instance_id'] = instance_id
return event_dict
# if fluentd is specified, we need to override the config data with
# its host and port info
if fluentd is not None:
fluentd_host = fluentd.split(':')[0].strip()
fluentd_port = int(fluentd.split(':')[1].strip())
handlers = log_config.get('handlers', None)
if isinstance(handlers, dict):
for _, defs in handlers.iteritems():
if isinstance(defs, dict):
if defs.get('class', '').endswith('FluentHandler'):
defs['host'] = fluentd_host
defs['port'] = fluentd_port
# Configure standard logging
logging.config.dictConfig(log_config)
logging.root.level -= 10 * verbosity_adjust
processors = [
add_exc_info_flag_for_exception,
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
add_instance_id,
FluentRenderer(),
]
structlog.configure(logger_factory=structlog.stdlib.LoggerFactory(),
context_class=PlainRenderedOrderedDict,
wrapper_class=BoundLogger,
processors=processors)
# Mark first line of log
log = structlog.get_logger()
log.info("first-line")
return log
def configure():
structlog.configure(
processors=[
structlog.processors.StackInfoRenderer(),
structlog.processors.TimeStamper(fmt='iso'),
structlog.twisted.JSONRenderer(),
],
context_class=dict,
logger_factory=structlog.twisted.LoggerFactory(),
wrapper_class=structlog.twisted.BoundLogger,
cache_logger_on_first_use=True,
)
def configure():
"""Configure Balanced logging system
"""
structlog.configure(
processors=[
format_exc_info,
LogProcessor(),
JSONRenderer(),
],
logger_factory=LoggerFactory(),
)
def includeme(config):
# Configure the standard library logging
logging.config.dictConfig({
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"structlog": {
"()": "warehouse.logging.StructlogFormatter",
},
},
"handlers": {
"primary": {
"class": "logging.StreamHandler",
"stream": "ext://sys.stdout",
"formatter": "structlog",
},
"sentry": {
"class": "raven.handlers.logging.SentryHandler",
"level": "ERROR",
"dsn": config.registry.settings.get("sentry.dsn"),
"release": config.registry.settings.get("warehouse.commit"),
"transport": config.registry.settings.get("sentry.transport"),
},
},
"root": {
"level": config.registry.settings.get("logging.level", "INFO"),
"handlers": ["primary", "sentry"],
},
})
# Configure structlog
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
RENDERER,
],
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
# Give every request a unique identifier
config.add_request_method(_create_id, name="id", reify=True)
# Add a log method to every request.
config.add_request_method(_create_logger, name="log", reify=True)
from buildscripts.evergreen_burn_in_tests import GenerateConfig, DEFAULT_PROJECT, CONFIG_FILE, \
EvergreenFileChangeDetector
from buildscripts.resmokelib.suitesconfig import get_named_suites_with_root_level_key
from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder
from buildscripts.task_generation.gen_config import GenerationConfiguration
from buildscripts.task_generation.generated_config import GeneratedConfiguration
from buildscripts.task_generation.multiversion_util import MultiversionUtilService
from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig
from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters
from buildscripts.task_generation.suite_split_strategies import SplitStrategy, greedy_division, \
FallbackStrategy, round_robin_fallback
from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions
from buildscripts.task_generation.task_types.multiversion_tasks import MultiversionGenTaskParams
from buildscripts.util.cmdutils import enable_logging
structlog.configure(logger_factory=LoggerFactory())
LOGGER = structlog.getLogger(__name__)
MULTIVERSION_CONFIG_KEY = gen_multiversion.MULTIVERSION_CONFIG_KEY
MULTIVERSION_PASSTHROUGH_TAG = gen_multiversion.PASSTHROUGH_TAG
BURN_IN_MULTIVERSION_TASK = gen_multiversion.BURN_IN_TASK
DEFAULT_CONFIG_DIR = "generated_resmoke_config"
DEFAULT_TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig",
"suites")
def filter_list(item: str, input_list: List[str]) -> bool:
"""
Filter to determine if the given item is in the given list.
:param item: Item to search for.
def __init__(self, connection=None, config=None, setup_structlog=False):
"""
Initializes TaskTiger with the given Redis connection and config
options. Optionally sets up structlog.
"""
self.config = {
# String that is used to prefix all Redis keys
'REDIS_PREFIX': 't',
# Name of the Python (structlog) logger
'LOGGER_NAME': 'tasktiger',
# Where to queue tasks that don't have an explicit queue
'DEFAULT_QUEUE': 'default',
# After how many seconds time out on listening on the activity
# channel and check for scheduled or expired items.
'SELECT_TIMEOUT': 1,
# If this is True, all tasks except future tasks (when=a future
# time) will be executed locally by blocking until the task
# returns. This is useful for testing purposes.
'ALWAYS_EAGER': False,
# If retry is True but no retry_method is specified for a given
# task, use the following default method.
'DEFAULT_RETRY_METHOD': fixed(60, 3),
# After how many seconds a task that can't require a lock is
# retried.
'LOCK_RETRY': 1,
# How many items to move at most from the scheduled queue to the
# active queue.
'SCHEDULED_TASK_BATCH_SIZE': 1000,
# After how many seconds a long-running task is killed. This can be
# overridden by the task or at queue time.
'DEFAULT_HARD_TIMEOUT': 300,
# The timer specifies how often the worker updates the task's
# timestamp in the active queue. Tasks exceeding the timeout value
# are requeued. Note that no delay is necessary before the retry
# since this condition happens when the worker crashes, and not
# when there is an exception in the task itself.
'ACTIVE_TASK_UPDATE_TIMER': 10,
'ACTIVE_TASK_UPDATE_TIMEOUT': 60,
'ACTIVE_TASK_EXPIRED_BATCH_SIZE': 10,
# Set up queues that will be processed in batch, i.e. multiple jobs
# are taken out of the queue at the same time and passed as a list
# to the worker method. Takes a dict where the key represents the
# queue name and the value represents the batch size. Note that the
# task needs to be declared as batch=True. Also note that any
# subqueues will be automatically treated as batch queues, and the
# batch value of the most specific subqueue name takes precedence.
'BATCH_QUEUES': {},
# How often to print stats.
'STATS_INTERVAL': 60,
}
if config:
self.config.update(config)
self.connection = connection or redis.Redis()
self.scripts = RedisScripts(self.connection)
if setup_structlog:
structlog.configure(
processors=[
structlog.stdlib.add_log_level,
structlog.stdlib.filter_by_level,
structlog.processors.TimeStamper(fmt='iso', utc=True),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.JSONRenderer()
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
self.log = structlog.get_logger(self.config['LOGGER_NAME'], ).bind()
if setup_structlog:
self.log.setLevel(logging.DEBUG)
logging.basicConfig(format='%(message)s')
def configure_structlog():
"""
Make structlog comply with all of our options.
"""
from django.conf import settings
import logging.config
import structlog
from sentry import options
from sentry.logging import LoggingFormat
WrappedDictClass = structlog.threadlocal.wrap_dict(dict)
kwargs = {
"context_class":
WrappedDictClass,
"wrapper_class":
structlog.stdlib.BoundLogger,
"cache_logger_on_first_use":
True,
"processors": [
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.format_exc_info,
structlog.processors.StackInfoRenderer(),
structlog.processors.UnicodeDecoder(),
],
}
fmt_from_env = os.environ.get("SENTRY_LOG_FORMAT")
if fmt_from_env:
settings.SENTRY_OPTIONS["system.logging-format"] = fmt_from_env.lower()
fmt = options.get("system.logging-format")
if fmt == LoggingFormat.HUMAN:
from sentry.logging.handlers import HumanRenderer
kwargs["processors"].extend(
[structlog.processors.ExceptionPrettyPrinter(),
HumanRenderer()])
elif fmt == LoggingFormat.MACHINE:
from sentry.logging.handlers import JSONRenderer
kwargs["processors"].append(JSONRenderer())
structlog.configure(**kwargs)
lvl = os.environ.get("SENTRY_LOG_LEVEL")
if lvl:
levelNames = logging._levelNames if not six.PY3 else logging._nameToLevel
if lvl not in levelNames:
raise AttributeError("%s is not a valid logging level." % lvl)
settings.LOGGING["root"].update(
{"level": lvl or settings.LOGGING["default_level"]})
if lvl:
for logger in settings.LOGGING["overridable"]:
try:
settings.LOGGING["loggers"][logger].update({"level": lvl})
except KeyError:
raise KeyError("%s is not a defined logger." % logger)
logging.config.dictConfig(settings.LOGGING)
import os
from configurations import Configuration, values
from structlog import configure
from structlog.stdlib import LoggerFactory
from . import mixins
configure(logger_factory=LoggerFactory())
def project_path(*path):
this_dir = os.path.dirname(os.path.realpath(__file__))
return os.path.join(this_dir, '..', *path)
class Base(mixins.DjangoLoggingMixin, Configuration):
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ADMINS = (('Alerts', '*****@*****.**'))
MANAGERS = ADMINS
AUTH_USER_MODEL = 'users.User'
CACHES = values.CacheURLValue('locmem:///')
DATABASE_DICT = values.DatabaseURLValue()
@property
def configure_logging(
logger_level_config: Dict[str, str] = None,
colorize: bool = True,
log_json: bool = False,
log_file: str = None,
disable_debug_logfile: bool = False,
):
structlog.reset_defaults()
if logger_level_config is None:
logger_level_config = {'': DEFAULT_LOG_LEVEL}
processors = [
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S"),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
]
formatter = 'colorized' if colorize and not log_file else 'plain'
if log_json:
formatter = 'json'
redact = redactor({
re.compile(r'\b(access_?token=)([a-z0-9_-]+)', re.I):
r'\1<redacted>',
})
_wrap_tracebackexception_format(redact)
log_handler = _get_log_handler(
formatter,
log_file,
)
if disable_debug_logfile:
combined_log_handlers = log_handler
else:
debug_log_file_handler = _get_log_file_handler()
combined_log_handlers = {**log_handler, **debug_log_file_handler}
logging.config.dictConfig(
{
'version': 1,
'disable_existing_loggers': False,
'filters': {
'log_level_filter': {
'()': RaidenFilter,
'log_level_config': logger_level_config,
},
'log_level_debug_filter': {
'()': RaidenFilter,
'log_level_config': {
'': DEFAULT_LOG_LEVEL,
'raiden': 'DEBUG'
},
},
},
'formatters': {
'plain': {
'()':
structlog.stdlib.ProcessorFormatter,
'processor':
_chain(structlog.dev.ConsoleRenderer(colors=False),
redact),
'foreign_pre_chain':
processors,
},
'json': {
'()':
structlog.stdlib.ProcessorFormatter,
'processor':
_chain(structlog.processors.JSONRenderer(), redact),
'foreign_pre_chain':
processors,
},
'colorized': {
'()':
structlog.stdlib.ProcessorFormatter,
'processor':
_chain(structlog.dev.ConsoleRenderer(colors=True), redact),
'foreign_pre_chain':
processors,
},
'debug': {
'()':
structlog.stdlib.ProcessorFormatter,
'processor':
_chain(structlog.dev.ConsoleRenderer(colors=False),
redact),
'foreign_pre_chain':
processors,
},
},
'handlers': combined_log_handlers,
'loggers': {
'': {
'handlers': list(combined_log_handlers.keys()),
'propagate': True,
},
},
}, )
structlog.configure(
processors=processors + [
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
wrapper_class=structlog.stdlib.BoundLogger,
logger_factory=structlog.stdlib.LoggerFactory(),
cache_logger_on_first_use=True,
)
# set logging level of the root logger to DEBUG, to be able to intercept
# all messages, which are then be filtered by the `RaidenFilter`
structlog.get_logger('').setLevel(
logger_level_config.get('', DEFAULT_LOG_LEVEL))
structlog.get_logger('raiden').setLevel('DEBUG')
# rollover RotatingFileHandler on startup, to split logs also per-session
root = logging.getLogger()
for handler in root.handlers:
if isinstance(handler, logging.handlers.RotatingFileHandler):
handler.flush()
if os.stat(handler.baseFilename).st_size > 0:
handler.doRollover()
# fix logging of py-evm (it uses a custom Trace logger from logging library)
# if py-evm is not used this will throw, hence the try-catch block
# for some reason it didn't work to put this into conftest.py
try:
from eth.tools.logging import setup_trace_logging
setup_trace_logging()
except ImportError:
pass
import math
import structlog
structlog.configure(logger_factory=structlog.PrintLoggerFactory())
logger = structlog.get_logger(processors=[structlog.processors.JSONRenderer()])
def add_equivalent_travel_cost(travel_options, value_one_hour,
value_ten_hours):
"""
Calculate the dollar equivalent of the travel time, add that to the monetary travel cost, and store the results
in a new column called `equivalent_travel_cost`
:param travel_options: pandas.DataFrame: each row is a travel option
:param value_one_hour:int: value in dollars of 1 hour of time
:param value_ten_hours: int: value in dollars of 10 hours of time
:return: pandas.DataFrame: travel_options with the new column `equivalent_travel_cost`
"""
# Formula used: Y = a(X^b), where X is the travel time in hours, and Y is the dollar equivalent value
a = value_one_hour
b = math.log(value_ten_hours / value_one_hour) / math.log(10)
logger.info("Calculating dollar equivalent of travel time",
formula="Y = a(X^b)",
a=a,
b=b)
def calculate_equivalent_travel_cost(row):
""" Calculate the equivalent travel cost for a single travel option """
travel_time_hours = row.travel_time_seconds / 3600
time_value = a * (travel_time_hours**b)
return row.total_cost + time_value
def configure_logging(
logger_level_config: Dict[str, str] = None,
colorize: bool = True,
log_json: bool = False,
log_file: str = None,
disable_debug_logfile: bool = False,
debug_log_file_name: str = None,
_first_party_packages: FrozenSet[str] = _FIRST_PARTY_PACKAGES,
cache_logger_on_first_use: bool = True,
):
structlog.reset_defaults()
logger_level_config = logger_level_config or dict()
logger_level_config.setdefault('filelock', 'ERROR')
logger_level_config.setdefault('', DEFAULT_LOG_LEVEL)
processors = [
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
add_greenlet_name,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S.%f"),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
]
if log_json:
formatter = 'json'
elif colorize and not log_file:
formatter = 'colorized'
else:
formatter = 'plain'
redact = redactor({
re.compile(r'\b(access_?token=)([a-z0-9_-]+)', re.I):
r'\1<redacted>',
})
_wrap_tracebackexception_format(redact)
handlers = dict()
if log_file:
handlers['file'] = {
'class': 'logging.handlers.WatchedFileHandler',
'filename': log_file,
'level': 'DEBUG',
'formatter': formatter,
'filters': ['user_filter'],
}
else:
handlers['default'] = {
'class': 'logging.StreamHandler',
'level': 'DEBUG',
'formatter': formatter,
'filters': ['user_filter'],
}
if not disable_debug_logfile:
if debug_log_file_name is None:
time = datetime.datetime.utcnow().isoformat()
debug_log_file_name = f'raiden-debug_{time}.log'
handlers['debug-info'] = {
'class': 'logging.handlers.RotatingFileHandler',
'filename': debug_log_file_name,
'level': 'DEBUG',
'formatter': 'debug',
'maxBytes': MAX_LOG_FILE_SIZE,
'backupCount': LOG_BACKUP_COUNT,
'filters': ['raiden_debug_file_filter'],
}
logging.config.dictConfig(
{
'version': 1,
'disable_existing_loggers': False,
'filters': {
'user_filter': {
'()': RaidenFilter,
'log_level_config': logger_level_config,
},
'raiden_debug_file_filter': {
'()': RaidenFilter,
'log_level_config': {
'': DEFAULT_LOG_LEVEL,
'raiden': 'DEBUG',
},
},
},
'formatters': {
'plain': {
'()':
structlog.stdlib.ProcessorFormatter,
'processor':
_chain(structlog.dev.ConsoleRenderer(colors=False),
redact),
'foreign_pre_chain':
processors,
},
'json': {
'()':
structlog.stdlib.ProcessorFormatter,
'processor':
_chain(structlog.processors.JSONRenderer(), redact),
'foreign_pre_chain':
processors,
},
'colorized': {
'()':
structlog.stdlib.ProcessorFormatter,
'processor':
_chain(structlog.dev.ConsoleRenderer(colors=True), redact),
'foreign_pre_chain':
processors,
},
'debug': {
'()':
structlog.stdlib.ProcessorFormatter,
'processor':
_chain(structlog.processors.JSONRenderer(), redact),
'foreign_pre_chain':
processors,
},
},
'handlers': handlers,
'loggers': {
'': {
'handlers': handlers.keys(),
'propagate': True,
},
},
}, )
structlog.configure(
processors=processors + [
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
wrapper_class=structlog.stdlib.BoundLogger,
logger_factory=structlog.stdlib.LoggerFactory(),
cache_logger_on_first_use=cache_logger_on_first_use,
)
# set logging level of the root logger to DEBUG, to be able to intercept
# all messages, which are then be filtered by the `RaidenFilter`
structlog.get_logger('').setLevel(
logger_level_config.get('', DEFAULT_LOG_LEVEL))
for package in _first_party_packages:
structlog.get_logger(package).setLevel('DEBUG')
# rollover RotatingFileHandler on startup, to split logs also per-session
root = logging.getLogger()
for handler in root.handlers:
if isinstance(handler, logging.handlers.RotatingFileHandler):
handler.flush()
if os.stat(handler.baseFilename).st_size > 0:
handler.doRollover()
# fix logging of py-evm (it uses a custom Trace logger from logging library)
# if py-evm is not used this will throw, hence the try-catch block
# for some reason it didn't work to put this into conftest.py
try:
from eth.tools.logging import setup_trace_logging
setup_trace_logging()
except ImportError:
pass
import logging
import sys
import structlog
from structlog import get_logger
logger = get_logger()
def hello():
logger.info("hello", name="foo", age=20)
return "hello"
def broken():
try:
return 1 / 0
except Exception as e:
logger.error(e, exc_info=True)
if __name__ == "__main__":
logging.basicConfig(
format="%(message)s %(asctime)s %(levelname)s)",
stream=sys.stdout,
level=logging.INFO,
)
structlog.configure(logger_factory=structlog.stdlib.LoggerFactory(), )
print(hello())
print(broken())
# Boundaries
BOUNDARIES_SHAPEFILES_DIR = "shapefiles"
# API
CORS_ORIGIN_ALLOW_ALL = True
CORS_URLS_REGEX = r"^/(graphql|api/v1).*$"
CORS_ALLOW_METHODS = ["GET", "POST", "OPTIONS"]
CORS_ALLOW_HEADERS = default_headers + ("x-api-key", )
GRAPHENE = {"SCHEMA": "graphapi.schema.schema", "MIDDLEWARE": []}
# structlog config
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt="iso"),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
structlog.processors.JSONRenderer(),
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
""" Experimenting with using structlog with hope of supporting it in Lightbus
"""
import logging
import sys
import structlog
def event_dict_ordering(logger, method_name, event_dict):
ordered = {"event": event_dict.pop("event")}
ordered.update(**event_dict)
return ordered
structlog.configure(processors=[
event_dict_ordering,
structlog.stdlib.add_log_level,
structlog.stdlib.add_logger_name,
structlog.processors.TimeStamper(fmt="iso"),
structlog.dev.ConsoleRenderer() if sys.stdout.isatty() else structlog.
processors.JSONRenderer(),
])
if __name__ == "__main__":
log = structlog.wrap_logger(logging.getLogger("test"))
log.warning("hello from std", foo=1)
log.info("Loaded plugins", plugins={...}, context={"service_name": "..."})
chain = [
LogEntryProcessor.add_app_info,
LogEntryProcessor.add_logger_name,
structlog.stdlib.add_log_level,
LogEntryProcessor.add_timestamp,
LogEntryProcessor.censor_password,
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
LogEntryProcessor.cleanup_keynames,
structlog.processors.JSONRenderer(),
]
structlog.configure(
processors=chain,
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
# structlog.configure(
# context_class=structlog.threadlocal.wrap_dict(dict),
# logger_factory=structlog.stdlib.LoggerFactory(),
# wrapper_class=structlog.stdlib.BoundLogger,
# processors=[
# structlog.stdlib.filter_by_level,
# structlog.stdlib.add_logger_name,
# structlog.stdlib.add_log_level,
# structlog.stdlib.add_log_level_number,
# structlog.stdlib.PositionalArgumentsFormatter(),
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt="iso"),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
]
structlog_post_processors = [
SentryJsonProcessor(level=logging.ERROR, tag_keys="__all__"),
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
]
structlog.configure(
processors=[structlog.stdlib.filter_by_level] + shared_processors +
structlog_post_processors,
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
# Logging config
# ==============
formatter = structlog.stdlib.ProcessorFormatter(
processor=structlog.processors.JSONRenderer(),
foreign_pre_chain=shared_processors)
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root_logger = logging.getLogger()
root_logger.addHandler(handler)
root_logger.setLevel(logging.INFO)
def set_default_append(self):
DEBUG = self.get("DEBUG")
self.set_option("DEBUG_EMAIL", DEBUG)
for template in self.get("TEMPLATES"):
template["OPTIONS"]["debug"] = DEBUG
# TEMPLATES[0]["OPTIONS"]["debug"] = DEBUG
# use structlog for logging
import structlog
MIDDLEWARE = self.get("MIDDLEWARE")
MIDDLEWARE += [
"django_structlog.middlewares.RequestMiddleware",
]
# set these explicitly, not with DEBUG
DJANGO_LOG_LEVEL = self.set_option("DJANGO_LOG_LEVEL", "INFO")
FULLCTL_LOG_LEVEL = self.set_option("FULLCTL_LOG_LEVEL", "DEBUG")
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.processors.TimeStamper(fmt="iso"),
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
context_class=structlog.threadlocal.wrap_dict(dict),
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
# logging define extra formatters and handlers for convenience
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"json": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": structlog.processors.JSONRenderer(),
},
"color_console": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": structlog.dev.ConsoleRenderer(),
},
"key_value": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": structlog.processors.KeyValueRenderer(
key_order=["timestamp", "level", "event", "logger"]
),
},
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"formatter": "color_console",
"stream": sys.stdout,
},
"console_json": {
"class": "logging.StreamHandler",
"formatter": "json",
"stream": sys.stdout,
},
"mail_admins": {
"class": "django.utils.log.AdminEmailHandler",
"level": "ERROR",
# plain text by default - HTML is nicer
"include_html": True,
},
},
"loggers": {
"django": {
"handlers": ["console_json"],
"level": DJANGO_LOG_LEVEL,
},
"django_structlog": {
"handlers": ["console_json"],
"level": FULLCTL_LOG_LEVEL,
},
},
}
self.set_option("LOGGING", LOGGING)
stdlib.ProcessorFormatter.wrap_for_formatter,
]
prod_processors = [
stdlib.filter_by_level,
stdlib.add_logger_name,
stdlib.add_log_level,
add_environment,
processors.format_exc_info,
processors.UnicodeDecoder(),
processors.TimeStamper(fmt="ISO", utc=True, key="@timestamp"),
SentryJsonProcessor(level=logging.ERROR, tag_keys=["environment"]),
stdlib.ProcessorFormatter.wrap_for_formatter,
]
if ENVIRONMENT in ("dev", "test"):
processors_list = dev_processors
else:
processors_list = prod_processors
sentry_sdk.init(
dsn=str(SENTRY_DSN),
environment=ENVIRONMENT,
)
configure(
processors=processors_list,
logger_factory=stdlib.LoggerFactory(),
wrapper_class=stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def configure_logging(
logger_level_config: Dict[str, str] = None,
colorize: bool = True,
log_json: bool = False,
log_file: str = None,
disable_debug_logfile: bool = False,
debug_log_file_name: str = None,
cache_logger_on_first_use: bool = True,
_first_party_packages: FrozenSet[str] = _FIRST_PARTY_PACKAGES,
_debug_log_file_additional_level_filters: Dict[str, str] = None,
) -> None:
structlog.reset_defaults()
logger_level_config = logger_level_config or dict()
logger_level_config.setdefault("filelock", "ERROR")
logger_level_config.setdefault("", DEFAULT_LOG_LEVEL)
processors = [
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
add_greenlet_name,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S.%f"),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
]
if log_json:
formatter = "json"
elif colorize and not log_file:
formatter = "colorized"
else:
formatter = "plain"
redact = redactor(LOG_BLACKLIST)
handlers: Dict[str, Any] = dict()
if log_file:
handlers["file"] = {
"class": "logging.handlers.WatchedFileHandler",
"filename": log_file,
"level": "DEBUG",
"formatter": formatter,
"filters": ["user_filter"],
}
else:
handlers["default"] = {
"class": "logging.StreamHandler",
"level": "DEBUG",
"formatter": formatter,
"filters": ["user_filter"],
}
if not disable_debug_logfile:
if debug_log_file_name is None:
time = datetime.datetime.utcnow().isoformat()
debug_log_file_name = f"raiden-debug_{time}.log"
handlers["debug-info"] = {
"class": "logging.handlers.RotatingFileHandler",
"filename": debug_log_file_name,
"level": "DEBUG",
"formatter": "debug",
"maxBytes": MAX_LOG_FILE_SIZE,
"backupCount": LOG_BACKUP_COUNT,
"filters": ["raiden_debug_file_filter"],
}
logging.config.dictConfig({
"version": 1,
"disable_existing_loggers": False,
"filters": {
"user_filter": {
"()": RaidenFilter,
"log_level_config": logger_level_config
},
"raiden_debug_file_filter": {
"()": RaidenFilter,
"log_level_config": {
"": DEFAULT_LOG_LEVEL,
"raiden": "DEBUG",
**(_debug_log_file_additional_level_filters or {}),
},
},
},
"formatters": {
"plain": {
"()":
structlog.stdlib.ProcessorFormatter,
"processor":
_chain(structlog.dev.ConsoleRenderer(colors=False), redact),
"foreign_pre_chain":
processors,
},
"json": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": _chain(structlog.processors.JSONRenderer(),
redact),
"foreign_pre_chain": processors,
},
"colorized": {
"()":
structlog.stdlib.ProcessorFormatter,
"processor":
_chain(structlog.dev.ConsoleRenderer(colors=True), redact),
"foreign_pre_chain":
processors,
},
"debug": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": _chain(structlog.processors.JSONRenderer(),
redact),
"foreign_pre_chain": processors,
},
},
"handlers": handlers,
"loggers": {
"": {
"handlers": handlers.keys(),
"propagate": True
}
},
})
structlog.configure(
processors=processors +
[structlog.stdlib.ProcessorFormatter.wrap_for_formatter],
wrapper_class=structlog.stdlib.BoundLogger,
logger_factory=structlog.stdlib.LoggerFactory(),
cache_logger_on_first_use=cache_logger_on_first_use,
)
# set logging level of the root logger to DEBUG, to be able to intercept
# all messages, which are then be filtered by the `RaidenFilter`
structlog.get_logger("").setLevel(
logger_level_config.get("", DEFAULT_LOG_LEVEL))
for package in _first_party_packages:
structlog.get_logger(package).setLevel("DEBUG")
# rollover RotatingFileHandler on startup, to split logs also per-session
root = logging.getLogger()
for handler in root.handlers:
if isinstance(handler, logging.handlers.RotatingFileHandler):
handler.flush()
if os.stat(handler.baseFilename).st_size > 0:
handler.doRollover()
# fix logging of py-evm (it uses a custom Trace logger from logging library)
# if py-evm is not used this will throw, hence the try-catch block
# for some reason it didn't work to put this into conftest.py
try:
from eth.tools.logging import setup_trace_logging
setup_trace_logging()
except ImportError:
pass
global IS_CONFIGURED, PROFILES, DOCKER_URL
IS_CONFIGURED = True
if isinstance(profiles, dict):
profiles_map = {
name: Profile(name, **profile_kwargs)
for name, profile_kwargs in profiles.items()
}
else:
profiles_map = {profile.name: profile for profile in profiles or []}
PROFILES.update(profiles_map)
DOCKER_URL = docker_url
if not structlog._config._CONFIG.is_configured:
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt='iso'),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.KeyValueRenderer(key_order=['event']),
],
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def default_logging(
verbosity: int,
log_format: LogFormat = LogFormat.TEXT,
external_logs: Optional[Iterable[str]] = None,
loggers_to_configure: Optional[Iterable[str]] = None,
) -> None:
"""
Configure structlog based on the given parameters.
Logging will be done to stdout.
:param verbosity: Amount of verbosity to use.
:param log_format: Format to logs should be written in.
:param external_logs: External modules that should have logging turned down unless verbosity is
set to highest level.
:param loggers_to_configure: Names of loggers to configure with the same configuration.
"""
level = Verbosity(verbosity).level()
if log_format == LogFormat.TEXT:
logging.basicConfig(level=level,
stream=sys.stdout,
format=TEXT_LOG_FORMAT)
structlog.configure(
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
)
elif log_format == LogFormat.JSON:
# Setup json logging.
logger_config = {"handlers": ["json"], "level": level}
loggers = build_loggers_dictionary(loggers_to_configure, logger_config)
logging.config.dictConfig({
"version": 1,
"formatters": {
"json": {
"format": "%(message)s $(lineno)d $(filename)s",
"class": "pythonjsonlogger.jsonlogger.JsonFormatter",
}
},
"handlers": {
"json": {
"class": "logging.StreamHandler",
"formatter": "json"
}
},
"loggers": loggers,
})
structlog.configure(
context_class=structlog.threadlocal.wrap_dict(dict),
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
structlog.stdlib.render_to_log_kwargs,
],
)
# Unless the user specifies higher verbosity than we have levels, turn down the log level
# for external libraries.
if external_logs and verbosity < Verbosity.MAX:
# Turn down logging for modules outside this project.
for logger in external_logs:
logging.getLogger(logger).setLevel(logging.WARNING)
def init(self, connection=None, config=None, setup_structlog=False):
"""Provide Redis connection and config when lazy initialization is used."""
if self.config is not None:
raise RuntimeError('TaskTiger was already initialized')
self.config = {
# String that is used to prefix all Redis keys
'REDIS_PREFIX': 't',
# Name of the Python (structlog) logger
'LOGGER_NAME': 'tasktiger',
# Where to queue tasks that don't have an explicit queue
'DEFAULT_QUEUE': 'default',
# After how many seconds time out on listening on the activity
# channel and check for scheduled or expired items. The batch
# timeout will delay the specified seconds after the first message
# to wait for additional messages, useful for very active systems.
# Appropriate values: 0 <= SELECT_BATCH_TIMEOUT <= SELECT_TIMEOUT
'SELECT_TIMEOUT': 1,
'SELECT_BATCH_TIMEOUT': 0,
# If this is True, all tasks except future tasks (when=a future
# time) will be executed locally by blocking until the task
# returns. This is useful for testing purposes.
'ALWAYS_EAGER': False,
# If retry is True but no retry_method is specified for a given
# task, use the following default method.
'DEFAULT_RETRY_METHOD': fixed(60, 3),
# After how many seconds a task that can't acquire a lock is
# retried.
'LOCK_RETRY': 1,
# How many items to move at most from the scheduled queue to the
# active queue.
'SCHEDULED_TASK_BATCH_SIZE': 1000,
# After how many seconds a long-running task is killed. This can be
# overridden by the task or at queue time.
'DEFAULT_HARD_TIMEOUT': 300,
# The timer specifies how often the worker updates the task's
# timestamp in the active queue (in seconds). Tasks exceeding the
# timeout value are requeued periodically. This may happen when a
# worker crashes or is killed.
'ACTIVE_TASK_UPDATE_TIMER': 10,
'ACTIVE_TASK_UPDATE_TIMEOUT': 60,
# How often we requeue expired tasks (in seconds), and how many
# expired tasks we requeue at a time. The interval also determines
# the lock timeout, i.e. it should be large enough to have enough
# time to requeue a batch of tasks.
'REQUEUE_EXPIRED_TASKS_INTERVAL': 30,
'REQUEUE_EXPIRED_TASKS_BATCH_SIZE': 10,
# Set up queues that will be processed in batch, i.e. multiple jobs
# are taken out of the queue at the same time and passed as a list
# to the worker method. Takes a dict where the key represents the
# queue name and the value represents the batch size. Note that the
# task needs to be declared as batch=True. Also note that any
# subqueues will be automatically treated as batch queues, and the
# batch value of the most specific subqueue name takes precedence.
'BATCH_QUEUES': {},
# How often to print stats.
'STATS_INTERVAL': 60,
# Single worker queues can reduce redis activity in some use cases
# by locking at the queue level instead of just at the task or task
# group level. These queues will only allow a single worker to
# access the queue at a time. This can be useful in environments
# with large queues and many worker processes that need aggressive
# locking techniques.
'SINGLE_WORKER_QUEUES': [],
# The following settings are only considered if no explicit queues
# are passed in the command line (or to the queues argument in the
# run_worker() method).
# If non-empty, a worker only processeses the given queues.
'ONLY_QUEUES': [],
# If non-empty, a worker excludes the given queues from processing.
'EXCLUDE_QUEUES': [],
# List of context manager instances that will be called in each
# forked child process. Useful to do things like close file handles
# or reinitialize crypto libraries.
'CHILD_CONTEXT_MANAGERS': [],
# Store traceback in execution history for failed tasks. This can
# increase Redis storage requirements and therefore can be disabled
# if that is a concern.
'STORE_TRACEBACKS': True,
}
if config:
self.config.update(config)
if setup_structlog:
structlog.configure(
processors=[
structlog.stdlib.add_log_level,
structlog.stdlib.filter_by_level,
structlog.processors.TimeStamper(fmt='iso', utc=True),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.JSONRenderer()
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
self.log = structlog.get_logger(
self.config['LOGGER_NAME'],
).bind()
if setup_structlog:
self.log.setLevel(logging.DEBUG)
logging.basicConfig(format='%(message)s')
self.connection = connection or redis.Redis(decode_responses=True)
self.scripts = RedisScripts(self.connection)
time.sleep(5)
# Save cell infosets
for slot in config.slots:
backend.put(inserted_cells[slot])
if __name__ == '__main__':
# Restrict log message to be above selected level
structlog.configure(
processors=[
structlog.threadlocal.merge_threadlocal,
structlog.contextvars.merge_contextvars,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M.%S"),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.dev.ConsoleRenderer()
],
wrapper_class=structlog.make_filtering_bound_logger(logging.INFO),
logger_factory=structlog.PrintLoggerFactory(file=sys.stderr)
)
load_plugins()
parser = argparse.ArgumentParser(description='Megacell charger')
parser.add_argument('--loglevel', choices=LOG_LEVEL_NAMES, default='INFO', help='Change log level')
_config_group(parser)
# These will be needed only for workflows
add_plugin_args(parser)
if env is not None:
event_kw["env"] = env
return super(BoundLogger,
self)._proxy_to_logger(method_name, event, *event_args,
**event_kw)
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.processors.TimeStamper(fmt="iso", utc=True),
structlog.processors.StackInfoRenderer(),
_record_level,
_safe_exc_info_renderer,
_safe_encoding_renderer,
_record_module,
structlog.processors.JSONRenderer(),
],
context_class=wrap_dict(dict),
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=BoundLogger,
cache_logger_on_first_use=True,
)
get_logger = structlog.get_logger
# Convenience map to let users set level with a string
LOG_LEVELS = {
"debug": logging.DEBUG,
"info": logging.INFO,
"warning": logging.WARNING,
"error": logging.ERROR,
"default": {
"level": "DEBUG",
"class": "logging.StreamHandler",
"formatter": "colored",
},
},
"loggers": {
"": {
"handlers": ["default"],
"level": "DEBUG",
"propagate": True,
},
}
})
structlog.configure(
processors=[
structlog.stdlib.add_logger_name,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
timestamper,
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
A returncode and a string containing the output of visualmetrics.py
"""
cmd = [
"/usr/bin/python",
str(visualmetrics_path), "--video",
str(job.video_path)
]
cmd.extend(options)
return run_command(log, cmd)
if __name__ == "__main__":
structlog.configure(
processors=[
structlog.processors.TimeStamper(fmt="iso"),
structlog.processors.format_exc_info,
structlog.dev.ConsoleRenderer(colors=False),
],
cache_logger_on_first_use=True,
)
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
"visual_metrics_options",
type=str,
metavar="VISUAL-METRICS-OPTIONS",
help="Options to pass to visualmetrics.py",
nargs="*",
)
reports = [
v1.reports[codeword].handler_class(config=config, backend=backend)
for codeword in config.reports
]
for infoset in selected_cells(config=config, backend=backend):
for report in reports:
report.process_cell(infoset=infoset)
for report in reports:
report.report()
if __name__ == "__main__":
structlog.configure(
wrapper_class=structlog.make_filtering_bound_logger(logging.INFO),
logger_factory=structlog.PrintLoggerFactory(file=sys.stderr))
load_plugins()
parser = argparse.ArgumentParser(description='Report on cells')
parser.add_argument('--loglevel',
choices=LOG_LEVEL_NAMES,
default='INFO',
help='Change log level')
add_plugin_args(parser)
add_backend_selection_args(parser)
add_cell_selection_args(parser)
# Then add arguments dependent on the loaded plugins
parser.add_argument('-R',
from django import forms
from django.forms import ModelForm
from django.core.exceptions import ValidationError
from django.contrib import messages
from django.db.models import Exists
from django.db.models import Q
from .models import Portfolio, Project, User
import logging
logging.basicConfig()
import structlog
from structlog import get_logger
from structlog.stdlib import LoggerFactory
structlog.configure(logger_factory=LoggerFactory())
structlog.configure(processors=[structlog.processors.JSONRenderer()])
logger = get_logger()
class EditProjectForm(ModelForm):
class Meta:
model = Project
fields = ['root_task']
project_title = forms.CharField(label='Project Title', max_length=200)
project_version = forms.CharField(label='Project Version',
max_length=200)
class AddProjectForm(ModelForm):
class Meta:
model = Project
"elasticsearch": {"level": "WARNING"},
"falcon_cors": {"level": "WARNING", "formatter": "json"},
"urllib3": {"level": "CRITICAL"},
"timy": {"level": "ERROR"},
},
}
)
configure(
context_class=threadlocal.wrap_dict(dict),
logger_factory=stdlib.LoggerFactory(),
wrapper_class=stdlib.BoundLogger,
processors=[
stdlib.filter_by_level,
stdlib.add_logger_name,
stdlib.add_log_level,
stdlib.PositionalArgumentsFormatter(),
processors.TimeStamper(fmt="iso"),
processors.StackInfoRenderer(),
processors.format_exc_info,
processors.UnicodeDecoder(),
stdlib.render_to_log_kwargs,
],
)
# import logging
# import logging.config
# import structlog
# def setup_logging():
def stdlib_log_config(level="DEBUG", log_format=DEFAULT_FORMAT, log_type="text"):
if level not in LOG_LEVELS:
raise ValueError("Invalid log level: {}".format(level))
if log_type not in LOG_TYPES:
raise ValueError("Invalid log type: {}".format(log_type))
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
# structlog.processors.UnicodeEncoder(),
structlog.stdlib.render_to_log_kwargs,
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
if log_type == "text":
formatter = {
"format": log_format,
}
else:
formatter = {
"()": "pythonjsonlogger.jsonlogger.JsonFormatter",
"format": log_format,
}
conf = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {"default": formatter,},
"handlers": {
"console": {
"level": level,
"class": "logging.StreamHandler",
"formatter": "default",
}
},
"loggers": {
"PIL": {"handlers": ["console"], "level": "WARN", "propagate": False,},
"chardet": {"handlers": ["console"], "level": "INFO", "propagate": False,},
"django": {"handlers": ["console"], "level": "INFO", "propagate": False,},
"matplotlib": {
"handlers": ["console"],
"level": "INFO",
"propagate": False,
},
"newrelic": {"handlers": ["console"], "level": "WARN", "propagate": False,},
"requests": {"handlers": ["console"], "level": "WARN", "propagate": False,},
"selenium": {"handlers": ["console"], "level": "WARN", "propagate": False,},
"urllib3": {"handlers": ["console"], "level": "WARN", "propagate": False,},
},
"root": {"handlers": ["console"], "level": level,},
}
return conf
'level': 'DEBUG',
},
'transactions': {
'handlers': ['console', 'sentry'],
'level': 'DEBUG',
},
'accounts': {
'handlers': ['console', 'sentry'],
'level': 'DEBUG',
},
}
}
structlog.configure(logger_factory=structlog.stdlib.LoggerFactory(),
processors=[
structlog.processors.UnicodeEncoder(),
KeyValueRenderer(),
])
REST_FRAMEWORK = {
'DEFAULT_PAGINATION_CLASS':
'rest_framework.pagination.LimitOffsetPagination',
'PAGE_SIZE':
20,
'DEFAULT_RENDERER_CLASSES': (
'rest_framework.renderers.JSONRenderer',
'rest_framework.renderers.BrowsableAPIRenderer',
)
}
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
def datlog(
*,
level=logging.DEBUG,
capture_warnings=True,
redirect_print=False,
tty=None,
user_config=None,
json_renderer=None,
hook_thread=True,
hook_process=True,
use_processors=DEFAULT_PROCESSORS,
use_foreign_processors=DEFAULT_FOREIGN_PROCESSORS
):
"""Setup struct logging.
:param tty: if `False` the log will appear in json format
:param level: the root logger level
:param redirect_print: hijacks stdout/err
:param capture_warnings: capture warnings
:param user_config: merge user config with default log config
:param json_renderer: a custom json renderer
"""
if isinstance(level, str):
level = logging.getLevelName(level.upper())
if json_renderer is None:
json_renderer = JSONRenderer(
serializer=lambda obj, **kwargs: json.dumps(
stringify_dict_keys(obj), **kwargs
)
)
if tty is None:
tty = sys.stdout.isatty()
console_renderer = structlog.dev.ConsoleRenderer()
renderer = console_renderer if tty else json_renderer
pre_chain = [PROCESSORS[x] for x in use_foreign_processors]
config = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"structured": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": renderer,
"foreign_pre_chain": pre_chain,
},
"structured_console": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": console_renderer,
"foreign_pre_chain": pre_chain,
},
"structured_json": {
"()": structlog.stdlib.ProcessorFormatter,
"processor": json_renderer,
"foreign_pre_chain": pre_chain,
},
},
"handlers": {
"default": {"class": "logging.StreamHandler", "formatter": "structured"}
},
"loggers": {"": {"handlers": ["default"], "level": level, "propagate": True}},
}
if user_config:
merge_dict(config, user_config)
logging.config.dictConfig(config)
logging.captureWarnings(capture_warnings)
processors = [PROCESSORS[x] for x in use_processors]
processors.append(structlog.stdlib.ProcessorFormatter.wrap_for_formatter)
structlog.configure(
processors=processors,
context_class=structlog.threadlocal.wrap_dict(dict),
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
if redirect_print:
# redirect stdio print
print_log = structlog.get_logger("print")
sys.stderr = StdioToLog(print_log)
sys.stdout = StdioToLog(print_log)
# log uncaught exceptions
sys.excepthook = uncaught_exception
if hook_thread:
install_thread_excepthook()
if hook_process:
install_process_excepthook()
logger = structlog.get_logger()
return logger
from server.api.api_v1.api import api_router
from server.api.error_handling import ProblemDetailException
from server.db import db
from server.db.database import DBSessionMiddleware
from server.exception_handlers.generic_exception_handlers import form_error_handler, problem_detail_handler
from server.forms import FormException
from server.settings import app_settings
from server.version import GIT_COMMIT_HASH
structlog.configure(
processors=[
structlog.processors.add_log_level,
structlog.processors.StackInfoRenderer(),
structlog.dev.set_exc_info,
structlog.processors.format_exc_info,
structlog.processors.TimeStamper(),
structlog.dev.ConsoleRenderer(),
],
wrapper_class=structlog.make_filtering_bound_logger(logging.NOTSET),
context_class=dict,
logger_factory=structlog.PrintLoggerFactory(),
cache_logger_on_first_use=False,
)
logger = structlog.get_logger(__name__)
app = FastAPI(
title="Boilerplate",
description="The boilerplate is a project that can be copied and adapted.",
openapi_url="/api/openapi.json",
docs_url="/api/docs",
redoc_url="/api/redoc",
'"message":"",'
'"message_json": %(message)s}')
}
},
'handlers': {
'json': {
'class': 'logging.StreamHandler',
'formatter': 'json'
}
},
'loggers': {
'app': {
'handlers': ['json'],
'propagate': False
}
}
})
configure(
context_class=threadlocal.wrap_dict(dict),
logger_factory=stdlib.LoggerFactory(),
wrapper_class=stdlib.BoundLogger,
processors=[
stdlib.filter_by_level,
stdlib.PositionalArgumentsFormatter(),
processors.StackInfoRenderer(),
processors.format_exc_info,
processors.JSONRenderer()]
)
