def __call__(self, logger, name, eventDict):
_stuff, _why, eventDict = _extractStuffAndWhy(eventDict)
if name == "err":
eventDict["event"] = _why
if isinstance(_stuff, Failure):
eventDict["exception"] = _stuff.getTraceback(detail="verbose")
_stuff.cleanFailure()
else:
eventDict["event"] = _why
return ((ReprWrapper(GenericJSONRenderer.__call__(self, logger, name, eventDict)),), {"_structlog": True})
def __call__(self, logger, name, eventDict):
_stuff, _why, eventDict = _extractStuffAndWhy(eventDict)
if name == 'err':
eventDict['event'] = _why
if isinstance(_stuff, Failure):
eventDict['exception'] = _stuff.getTraceback(detail='verbose')
_stuff.cleanFailure()
else:
eventDict['event'] = _why
return ((GenericJSONRenderer.__call__(self, logger, name, eventDict),),
{'_structlog': True})
def configure_logging(self):
if self.app.testing:
structlog.reset_defaults()
disabled = [
"docker.utils.config",
"docker.auth",
"docker.api.build",
"docker.api.swarm",
"docker.api.image",
"werkzeug",
"requests",
"urllib3",
]
for logger in disabled:
log = logging.getLogger(logger)
log.setLevel(logging.ERROR)
log.disabled = True
self.app.logger.disabled = True
logging.basicConfig(
level=self.log_level, stream=sys.stdout, format="%(message)s"
)
chain = [
filter_by_level,
add_log_level,
add_logger_name,
TimeStamper(fmt="iso"),
StackInfoRenderer(),
format_exc_info,
JSONRenderer(indent=1, sort_keys=True),
]
logger = logging.getLogger(__name__)
if self.testing:
chain = []
logger = structlog.ReturnLogger()
log = structlog.wrap_logger(
logger,
processors=chain,
context_class=dict,
wrapper_class=structlog.stdlib.BoundLogger,
# cache_logger_on_first_use=True,
)
self.logger = log
self.app.logger = self.logger
def setup_logging(args):
global log
logging.basicConfig(stream=sys.stdout, format="%(message)s")
def add_timestamp(_, __, event_dict):
event_dict["timestamp"] = datetime.datetime.utcnow()
return event_dict
log = structlog.wrap_logger(logging.getLogger(__name__),
processors=[
add_timestamp,
JSONRenderer(indent=1, sort_keys=True),
])
def logger_initial_config(service_name=None,
log_level=None,
logger_format=None,
logger_date_format=None):
if not logger_date_format:
logger_date_format = os.getenv("LOGGING_DATE_FORMAT",
"%Y-%m-%dT%H:%M%s")
if not log_level:
log_level = os.getenv("LOGGING_LEVEL")
if not logger_format:
logger_format = "%(message)s"
try:
indent = int(os.getenv("JSON_INDENT_LOGGING"))
except TypeError:
indent = None
except ValueError:
indent = None
def add_service(logger, method_name, event_dict): # pylint: disable=unused-argument
"""
Add the service name to the event dict.
"""
event_dict["service"] = service_name
return event_dict
def add_severity_level(logger, method_name, event_dict):
"""
Add the log level to the event dict.
"""
if method_name == "warn":
# The stdlib has an alias
method_name = "warning"
event_dict["severity"] = method_name
return event_dict
logging.basicConfig(stream=sys.stdout,
level=log_level,
format=logger_format)
configure(processors=[
add_severity_level,
add_log_level,
filter_by_level,
add_service,
format_exc_info,
TimeStamper(fmt=logger_date_format, utc=True, key="created_at"),
JSONRenderer(indent=indent),
])
def logger_initial_config(log_level='INFO',
logger_format="%(message)s",
logger_date_format="%Y-%m-%dT%H:%M%s"):
def add_service(logger, method_name, event_dict): # pylint: disable=unused-argument
"""
Add the service name to the event dict.
This adds `service: 'ras-frontstage'` to all log lines.
"""
event_dict['service'] = 'ras-frontstage'
return event_dict
logging.basicConfig(stream=sys.stdout,
level=log_level,
format=logger_format)
auth_log = logging.getLogger(__name__)
auth_log.addHandler(logging.NullHandler())
auth_log.propagate = False
def add_severity_level(logger, method_name, event_dict):
"""
Add the log level to the event dict.
"""
if method_name == "warn":
# The stdlib has an alias
method_name = "warning"
event_dict["severity"] = method_name
return event_dict
def parse_exception(_, __, event_dict):
exception = event_dict.get('exception')
if exception:
event_dict['exception'] = exception.replace("\"", "'").split("\n")
return event_dict
# setup file logging
renderer_processor = JSONRenderer(indent=None)
processors = [
add_severity_level, add_log_level, filter_by_level, add_service,
format_exc_info,
TimeStamper(fmt=logger_date_format, utc=True,
key='created_at'), parse_exception, renderer_processor
]
configure(context_class=wrap_dict(dict),
logger_factory=LoggerFactory(),
processors=processors,
cache_logger_on_first_use=True)
def logger_initial_config(service_name=None,
log_level=None,
logger_format=None,
logger_date_format=None):
# pylint: skip-file
if not logger_date_format:
logger_date_format = os.getenv('LOGGING_DATE_FORMAT',
"%Y-%m-%dT%H:%M%s")
if not log_level:
log_level = os.getenv('LOGGING_LEVEL')
if not logger_format:
logger_format = "%(message)s"
try:
indent = int(os.getenv('JSON_INDENT_LOGGING'))
except TypeError:
indent = None
except ValueError:
indent = None
def add_service(logger, method_name, event_dict): # pylint: disable=unused-argument
"""
Add the service name to the event dict.
"""
event_dict['service'] = service_name
return event_dict
def zipkin_ids(logger, method_name, event_dict):
event_dict['trace'] = ''
event_dict['span'] = ''
event_dict['parent'] = ''
if not flask.has_app_context():
return event_dict
if '_zipkin_span' not in g:
return event_dict
event_dict['span'] = g._zipkin_span.zipkin_attrs.span_id
event_dict['trace'] = g._zipkin_span.zipkin_attrs.trace_id
event_dict['parent'] = g._zipkin_span.zipkin_attrs.parent_span_id
return event_dict
logging.basicConfig(stream=sys.stdout,
level=log_level,
format=logger_format)
configure(processors=[
zipkin_ids, add_log_level, filter_by_level, add_service,
format_exc_info,
TimeStamper(fmt=logger_date_format, utc=True, key="created_at"),
JSONRenderer(indent=indent)
])
def __init__(self, level=logging.INFO, **kwargs):
structlog.configure(processors=[
structlog.processors.StackInfoRenderer(),
structlog.dev.set_exc_info,
structlog.processors.format_exc_info,
structlog.dev.ConsoleRenderer(),
])
self._logger = wrap_logger(
structlog.get_logger(),
processors=[TimeStamper(fmt="iso"),
JSONRenderer(sort_keys=True)],
)
self.level = level
self.bind(**kwargs)
self.events = []
self.persist = False
def logger_initial_config(log_level=None):
"""Configures the logger"""
service_name = "ras-rm-auth-service"
logger_date_format = os.getenv("LOGGING_DATE_FORMAT", "%Y-%m-%dT%H:%M%s")
logger_format = "%(message)s"
if not log_level:
log_level = os.getenv("SMS_LOG_LEVEL", "INFO")
try:
indent = int(os.getenv("JSON_INDENT_LOGGING"))
except (TypeError, ValueError):
indent = None
def add_service(logger, method_name, event_dict): # pylint: disable=unused-argument
"""
Add the service name to the event dict.
"""
event_dict["service"] = service_name
return event_dict
def add_severity_level(logger, method_name, event_dict): # pylint: disable=unused-argument
"""
Add the log level to the event dict.
"""
if method_name == "warn":
# The stdlib has an alias
method_name = "warning"
event_dict["severity"] = method_name
return event_dict
logging.basicConfig(stream=sys.stdout,
level=log_level,
format=logger_format)
configure(processors=[
add_severity_level,
add_log_level,
filter_by_level,
add_service,
TimeStamper(fmt=logger_date_format, utc=True, key="created_at"),
JSONRenderer(indent=indent),
])
oauth_log = logging.getLogger("requests_oauthlib")
oauth_log.addHandler(logging.NullHandler())
oauth_log.propagate = False
def configure_logging():
log_level = logging.INFO
debug = os.getenv("FLASK_ENV") == "development"
if debug:
log_level = logging.DEBUG
log_handler = logging.StreamHandler(sys.stdout)
log_handler.setLevel(log_level)
log_handler.addFilter(lambda record: record.levelno <= logging.WARNING)
error_log_handler = logging.StreamHandler(sys.stderr)
error_log_handler.setLevel(logging.ERROR)
logging.basicConfig(level=log_level,
format="%(message)s",
handlers=[error_log_handler, log_handler])
# Set werkzeug logging level
werkzeug_logger = logging.getLogger("werkzeug")
werkzeug_logger.setLevel(level=log_level)
def parse_exception(_, __, event_dict):
if debug:
return event_dict
exception = event_dict.get("exception")
if exception:
event_dict["exception"] = exception.replace('"', "'").split("\n")
return event_dict
# setup file logging
renderer_processor = (ConsoleRenderer() if debug else JSONRenderer(
serializer=json_dumps))
processors = [
add_log_level,
TimeStamper(key="created", fmt="iso"),
add_service,
format_exc_info,
parse_exception,
renderer_processor,
]
configure(
context_class=wrap_dict(dict),
logger_factory=LoggerFactory(),
processors=processors,
cache_logger_on_first_use=True,
)
def logger_initial_config(
service_name=None, log_level=None, logger_format=None, logger_date_format=None
):
if not logger_date_format:
logger_date_format = os.getenv("LOGGING_DATE_FORMAT", "%Y-%m-%dT%H:%M%s")
if not log_level:
log_level = os.getenv("SMS_LOG_LEVEL", "DEBUG")
if not logger_format:
logger_format = "%(message)s"
if not service_name:
service_name = os.getenv("NAME", "census-rm-case-service")
try:
indent = int(os.getenv("JSON_INDENT_LOGGING"))
except TypeError:
indent = None
except ValueError:
indent = None
def add_service(_1, _2, event_dict):
"""
Add the service name to the event dict.
"""
event_dict["service"] = service_name
return event_dict
def add_thread(_1, _2, event_dict):
"""Add the thread name to the event dict.
"""
thread_name = threading.currentThread().getName()
event_dict["data"] = {
"thread_name": thread_name
}
return event_dict
logging.basicConfig(stream=sys.stdout, level=log_level, format=logger_format)
configure(
processors=[
add_log_level,
filter_by_level,
add_service,
add_thread,
TimeStamper(fmt=logger_date_format, utc=True, key="created_at"),
JSONRenderer(indent=indent),
]
)
def __init__(self, output=None, source=None, namespace=None, service=None):
log = wrap_logger(PrintLogger(output),
processors=[
add_timestamp,
order_fields,
JSONRenderer(),
render_wrapp_log,
])
if not namespace:
try:
frame = inspect.currentframe(1)
namespace = frame.f_globals['__name__']
except:
namespace = 'unknown'
service = service or os.environ.get('SERVICE_NAME')
self._log = log.bind(namespace=namespace, service=service)
def basic_setup():
structlog.configure(
processors=[
filter_by_level, # for performance reasons
JSONRenderer(),
],
context_class=dict,
logger_factory=LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
global _logger
h = lg.StreamHandler()
h.setFormatter(
StructFormatter('%(asctime)s [%(process)d]: %(message)s',
datefmt='%d %H:%M:%S'))
_logger = Logger()
_logger.add_handler(h)
def setup(sentry: str, debug: bool = False) -> None:
processors = [
filter_by_level,
add_log_level,
add_logger_name,
PositionalArgumentsFormatter(),
StackInfoRenderer(),
format_exc_info,
UnicodeDecoder(),
]
configure(
logger_factory=LoggerFactory(),
wrapper_class=BoundLogger,
cache_logger_on_first_use=True,
)
if debug:
styles = ConsoleRenderer.get_default_level_styles()
styles["debug"] = DIM
processors += [
TimeStamper(fmt="%Y-%m-%d %H:%M:%S"),
ConsoleRenderer(level_styles=styles),
]
else:
handler = StreamHandler()
formatter = CustomJsonFormatter("%(levelname)s %(name)s %(message)s")
handler.setFormatter(formatter)
for module in ("tornado", "tortoise", "aiomysql"):
getLogger(module).addHandler(handler)
sentry_logging = LoggingIntegration(level=INFO, event_level=ERROR)
init(sentry, integrations=[sentry_logging])
processors.append(JSONRenderer())
handler = StreamHandler()
configure(processors=processors)
log = get_logger("api")
log.addHandler(handler)
log.propagate = False
log.setLevel(DEBUG if debug else INFO)
def configure_logging():
# set up some sane logging, as opposed to what flask does by default
log_format = "%(message)s"
levels = {
'CRITICAL': logging.CRITICAL,
'ERROR': logging.ERROR,
'WARNING': logging.WARNING,
'INFO': logging.INFO,
'DEBUG': logging.DEBUG,
}
handler = logging.StreamHandler()
logging.basicConfig(level=levels[EQ_LOG_LEVEL],
format=log_format,
handlers=[handler])
# turn boto logging to critical as it logs far too much and it's only used
# for cloudwatch logging
logging.getLogger("botocore").setLevel(logging.ERROR)
if EQ_CLOUDWATCH_LOGGING:
_setup_cloud_watch_logging()
# Set werkzeug logging level
werkzeug_logger = logging.getLogger('werkzeug')
werkzeug_logger.setLevel(level=levels[EQ_WERKZEUG_LOG_LEVEL])
# setup file logging
rotating_log_file = RotatingFileHandler(filename="eq.log",
maxBytes=1048576,
backupCount=10)
logging.getLogger().addHandler(rotating_log_file)
renderer_processor = ConsoleRenderer(
) if EQ_DEVELOPER_LOGGING else JSONRenderer()
processors = [
add_log_level,
TimeStamper(key='created', fmt='iso'), add_service, format_exc_info,
renderer_processor
]
configure(context_class=wrap_dict(dict),
logger_factory=LoggerFactory(),
processors=processors,
cache_logger_on_first_use=True)
def logging_setup(log_level='INFO'):
"""
Set up standard structlog logger
Args:
log_level: string, defined the logging level. Can be: 'INFO', 'WARNING'
Returns:
logger: instantiated logger
"""
# logging setup. Import log level from config.json
logging.basicConfig(stream=sys.stdout,
format="%(message)s",
level=log_level)
logger = wrap_logger(logging.getLogger(__name__),
processors=[
filter_by_level, add_timestamp,
JSONRenderer(indent=1, sort_keys=True)
])
return logger
def configure_logging():
# set up some sane logging, as opposed to what flask does by default
log_format = "%(message)s"
levels = {
'CRITICAL': logging.CRITICAL,
'ERROR': logging.ERROR,
'WARNING': logging.WARNING,
'INFO': logging.INFO,
'DEBUG': logging.DEBUG,
}
handler = logging.StreamHandler()
logging.basicConfig(level=levels[EQ_LOG_LEVEL],
format=log_format,
handlers=[handler])
# Set werkzeug logging level
werkzeug_logger = logging.getLogger('werkzeug')
werkzeug_logger.setLevel(level=levels[EQ_WERKZEUG_LOG_LEVEL])
def parse_exception(_, __, event_dict):
if EQ_DEVELOPER_LOGGING:
return event_dict
exception = event_dict.get('exception')
if exception:
event_dict['exception'] = exception.replace("\"", "'").split("\n")
return event_dict
# setup file logging
renderer_processor = ConsoleRenderer(
) if EQ_DEVELOPER_LOGGING else JSONRenderer()
processors = [
add_log_level,
TimeStamper(key='created', fmt='iso'), add_service, format_exc_info,
parse_exception, renderer_processor
]
configure(context_class=wrap_dict(dict),
logger_factory=LoggerFactory(),
processors=processors,
cache_logger_on_first_use=True)
def test_other_handlers_get_original_record(self, configure_for_pf,
capsys):
"""
Logging handlers that come after the handler with ProcessorFormatter
should receive original, unmodified record.
"""
configure_logging(None)
handler1 = logging.StreamHandler()
handler1.setFormatter(ProcessorFormatter(JSONRenderer()))
handler2 = type("", (), {})()
handler2.handle = call_recorder(lambda record: None)
handler2.level = logging.INFO
logger = logging.getLogger()
logger.addHandler(handler1)
logger.addHandler(handler2)
logger.info("meh")
assert 1 == len(handler2.handle.calls)
handler2_record = handler2.handle.calls[0].args[0]
assert "meh" == handler2_record.msg
def _configure_logger(level='INFO', indent=None):
logging.basicConfig(stream=sys.stdout, level=level, format='%(message)s')
try:
indent = int(os.getenv('LOGGING_JSON_INDENT') or indent)
except TypeError:
indent = None
except ValueError:
indent = None
def add_service(_, __, event_dict):
"""
Add the service name to the event dict.
"""
event_dict['service'] = os.getenv('NAME', 'sdc-responses-dashboard')
return event_dict
renderer_processor = JSONRenderer(indent=indent)
processors = [add_log_level, filter_by_level, add_service, format_exc_info, add_logger_name,
TimeStamper(fmt='%Y-%m-%dT%H:%M%s', utc=True, key='created_at'), renderer_processor]
structlog.configure(context_class=wrap_dict(dict), logger_factory=LoggerFactory(), processors=processors,
cache_logger_on_first_use=True)
def logger_initial_config():
def add_service(_1, _2, event_dict):
"""
Add the service name to the event dict.
"""
event_dict["service"] = Config.NAME
return event_dict
def add_log_severity(_1, method_name, event_dict):
"""
Add the logging level to the event dict as 'severity'
"""
if method_name == "warn":
# The stdlib has an alias, we always want 'warning' in full
method_name = "warning"
if method_name == "exception":
# exception level is not as universal, use 'error' instead
method_name = "error"
event_dict["severity"] = method_name
return event_dict
logging.basicConfig(stream=sys.stdout, level=Config.LOG_LEVEL, format="%(message)s")
configure(
processors=[
add_log_level,
add_log_severity,
filter_by_level,
add_service,
TimeStamper(fmt=Config.LOG_DATE_FORMAT, utc=True, key="created_at"),
JSONRenderer(),
]
)
logging.getLogger('pika').setLevel(Config.LOG_LEVEL_PIKA)
logging.getLogger('paramiko').setLevel(Config.LOG_LEVEL_PARAMIKO)
def logger_initial_config():
logger_date_format = os.getenv('LOGGING_DATE_FORMAT', "%Y-%m-%dT%H:%M%s")
logger_format = "%(message)s"
log_level = os.getenv('LOGGING_LEVEL', 'DEBUG')
service_name = 'ras-secure-message'
try:
indent = int(os.getenv('JSON_INDENT_LOGGING'))
except (TypeError, ValueError):
indent = None
def add_service(_1, _2, event_dict):
"""
Add the service name to the event dict.
"""
event_dict['service'] = service_name
return event_dict
def add_severity_level(logger, method_name, event_dict): # pylint: disable=unused-argument
"""
Add the log level to the event dict.
"""
if method_name == "warn":
# The stdlib has an alias
method_name = "warning"
event_dict["severity"] = method_name
return event_dict
logging.basicConfig(stream=sys.stdout,
level=log_level,
format=logger_format)
configure(processors=[
add_severity_level, add_log_level, filter_by_level, add_service,
TimeStamper(fmt=logger_date_format, utc=True, key="created_at"),
JSONRenderer(indent=indent)
])
def test_formatter_unsets_exc_info(self, configure_for_pf, capsys, keep):
"""
Stack traces doesn't get printed outside of the json document when
keep_exc_info are set to False but preserved if set to True.
"""
configure_logging(None)
logger = logging.getLogger()
def format_exc_info_fake(logger, name, event_dict):
event_dict = collections.OrderedDict(event_dict)
del event_dict["exc_info"]
event_dict["exception"] = "Exception!"
return event_dict
formatter = ProcessorFormatter(
processor=JSONRenderer(),
keep_stack_info=keep,
keep_exc_info=keep,
foreign_pre_chain=[format_exc_info_fake],
)
logger.handlers[0].setFormatter(formatter)
try:
raise RuntimeError("oh noo")
except Exception:
logging.getLogger().exception("seen worse")
out, err = capsys.readouterr()
assert "" == out
if keep is False:
assert (
'{"event": "seen worse", "exception": "Exception!"}\n'
) == err
else:
assert "Traceback (most recent call last):" in err
async def test_integration(self, capsys):
"""
Configure and log an actual entry.
"""
configure(
processors=[add_log_level, JSONRenderer()],
logger_factory=PrintLogger,
wrapper_class=AsyncBoundLogger,
cache_logger_on_first_use=True,
)
logger = get_logger()
await logger.bind(foo="bar").info("baz", x="42")
assert {
"foo": "bar",
"x": "42",
"event": "baz",
"level": "info",
} == json.loads(capsys.readouterr().out)
reset_defaults()
def logger_initial_config(service_name=None,
log_level=None,
logger_format=None,
logger_date_format=None):
if not logger_date_format:
logger_date_format = os.getenv('LOGGING_DATE_FORMAT',
"%Y-%m-%dT%H:%M%s")
if not log_level:
log_level = os.getenv('LOGGING_LEVEL')
if not logger_format:
logger_format = "%(message)s"
if not service_name:
service_name = os.getenv('NAME')
try:
indent = int(os.getenv('JSON_INDENT_LOGGING'))
except TypeError:
indent = None
except ValueError:
indent = None
def add_service(logger, method_name, event_dict): # pylint: disable=unused-argument
"""
Add the service name to the event dict.
"""
event_dict['service'] = service_name
return event_dict
logging.basicConfig(stream=sys.stdout,
level=log_level,
format=logger_format)
configure(processors=[
add_log_level, filter_by_level, add_service,
TimeStamper(fmt=logger_date_format, utc=True, key="created_at"),
JSONRenderer(indent=indent)
])
def test_formatter_unsets_stack_info(self, configure_for_pf, capsys, keep):
"""
Stack traces doesn't get printed outside of the json document when
keep_stack_info are set to False but preserved if set to True.
"""
configure_logging(None)
logger = logging.getLogger()
formatter = ProcessorFormatter(
processor=JSONRenderer(),
keep_stack_info=keep,
keep_exc_info=keep,
foreign_pre_chain=[],
)
logger.handlers[0].setFormatter(formatter)
logging.getLogger().warn("have a stack trace", stack_info=True)
out, err = capsys.readouterr()
assert "" == out
if keep is False:
assert 1 == err.count("Stack (most recent call last):")
else:
assert 2 == err.count("Stack (most recent call last):")
def get_logger(name="blackopt", logdir=LOGDIR, **initial_values):
path = os.path.join(get_rootdir(), logdir, name)
os.makedirs(os.path.dirname(path), exist_ok=True)
def rotating_file_handler(log_path):
return RotatingFileHandler(
log_path,
mode="a",
maxBytes=BACKUP_COUNT * MAX_FILE_SIZE,
backupCount=BACKUP_COUNT,
)
file_hdlr = rotating_file_handler(path)
if os.path.exists(path):
try:
if os.stat(path).st_size != 0:
file_hdlr.doRollover()
except FileNotFoundError:
# Rollover failed. prefer to overwrite old log instead terminating Moonfish.
file_hdlr = rotating_file_handler(path)
logger = logging.getLogger(name)
logger.addHandler(file_hdlr)
logger.setLevel(logging.INFO)
log = wrap_logger(
logger,
processors=[
# filter_by_level,
TimeStamper(fmt="iso"),
JSONRenderer(sort_keys=True),
],
**initial_values)
return log
from structlog import wrap_logger
from structlog.processors import JSONRenderer
import luigi
from luigi.local_target import LocalFileSystem
from wagl.acquisition import acquisitions
from wagl.singlefile_workflow import DataStandardisation
from tesp.package import package, PATTERN2, ARD
from tesp.constants import ProductPackage
from eugl.fmask import fmask
ERROR_LOGGER = wrap_logger(logging.getLogger('errors'),
processors=[JSONRenderer(indent=1, sort_keys=True)])
STATUS_LOGGER = wrap_logger(
logging.getLogger('status'),
processors=[JSONRenderer(indent=1, sort_keys=True)])
INTERFACE_LOGGER = logging.getLogger('luigi-interface')
@luigi.Task.event_handler(luigi.Event.FAILURE)
def on_failure(task, exception):
"""Capture any Task Failure here."""
ERROR_LOGGER.error(task=task.get_task_family(),
params=task.to_str_params(),
level1=getattr(task, 'level1', ''),
exception=exception.__str__(),
traceback=traceback.format_exc().splitlines())
pytz.utc).isoformat()
event_dict['level'] = level
if session:
event_dict['session_id'] = session.get('session_id')
if request:
try:
event_dict['ip_address'] = request.headers[
'X-Forwarded-For'].split(',')[0].strip()
except:
event_dict['ip_address'] = 'unknown'
return event_dict
# Add a handler to write log messages to a file
if app.config.get('LOG_FILE'):
file_handler = RotatingFileHandler(filename=app.config['LOG_FILENAME'],
maxBytes=app.config['LOG_MAXBYTES'],
backupCount=app.config['LOG_BACKUPS'],
mode='a',
encoding='utf-8')
file_handler.setLevel(logging.DEBUG)
app.logger.addHandler(file_handler)
# Wrap the application logger with structlog to format the output
logger = wrap_logger(app.logger,
processors=[add_fields,
JSONRenderer(indent=None)])
KeyValueRenderer)
from tzlocal import get_localzone
from . import DEBUG
from .middleware import CORS, TrailingSlash
from .storage import Store
try:
from logging.config import dictConfig
except ImportError:
from logutils.dictconfig import dictConfig
if DEBUG:
processors = (format_exc_info, KeyValueRenderer())
else:
processors = (format_exc_info, JSONRenderer())
logger = structlog.get_logger()
default_conf = {
'search_index':
'/srv/graphite/index',
'finders': [
'graphite_api.finders.whisper.WhisperFinder',
],
'functions': [
'graphite_api.functions.SeriesFunctions',
'graphite_api.functions.PieFunctions',
],
'whisper': {
'directories': [
def handleLogRecord(self, record):
logger = wrap_logger(
logging.getLogger(__name__),
processors=[add_timestamp,
JSONRenderer(sort_keys=True)])
logger.info(record)
def __init__(self, message, case_number):
#Logging setup for json only
self.message = message
self.timestamp = int(time.time())
self.datetime = datetime.utcnow().isoformat()
self.desc = "AWS_IR Action"
configure(processors=[JSONRenderer(indent=1, sort_keys=True)],
context_class=structlog.threadlocal.wrap_dict(dict),
logger_factory=structlog.stdlib.LoggerFactory())
self.log = get_logger('aws_ir.json')
event = ReturnLogger().msg('message',
message=self.message,
timestamp=self.timestamp,
datetime=self.datetime,
desc=self.desc)
def generate_log_filename(case_number):
filename = ("/tmp/{case_number}-aws_ir.log").format(
case_number=case_number)
return filename
def log_file_exists():
exists = os.path.isfile(self.logfile)
return exists
self.logfile = generate_log_filename(case_number)
def file_len(fname):
i = 0
with open(fname) as f:
for i, l in enumerate(f):
pass
return i + 1
def log_file_contains_events(logfile):
length = file_len(logfile)
if length >= 2:
return True
else:
return False
def stub_ts_file(logfile):
with open(logfile, "w") as f:
f.write("[ \n")
f.close()
def write_log_event(event):
logfile = self.logfile
if log_file_exists():
if (log_file_contains_events(logfile) == True):
f = open(logfile)
lines = f.readlines()
f.close()
with open(logfile, 'w') as w:
w.writelines([item for item in lines[:-1]])
w.write("\t" + str(event) + ",")
w.write("\n")
w.write("]")
w.close()
else:
stub_ts_file(logfile)
with open(logfile, 'a') as w:
w.write("\t" + str(event) + ",")
w.write("\n")
w.write("]")
w.close()
else:
stub_ts_file(logfile)
with open(logfile, 'a') as w:
w.write("\t" + str(event) + ",")
w.write("\n")
w.write("]")
w.close()
write_log_event(event[1])
from tqdm import tqdm
from audioutils.parallel import do_parallel_with_pbar
from audioutils.io import get_and_make_artist_and_album_dirs
from audioutils.conversion import convert_album
logfile_name = 'manage_player_log_{}.txt'.format(datetime.datetime.now())
logfile_path = os.path.join(pathlib.Path.home(), '.audioutils', 'logs',
logfile_name)
logging.basicConfig(filename=logfile_path, level=logging.DEBUG)
from structlog.stdlib import LoggerFactory
from structlog.processors import (JSONRenderer, TimeStamper)
structlog.configure(logger_factory=LoggerFactory(),
processors=[TimeStamper(), JSONRenderer()])
LOGGER = structlog.get_logger()
def convert_and_save_album(source, target, target_format, bitrate,
copy_non_sound, num_processes):
print('Inside convert_and_save_album', file=sys.stderr)
(artist, album) = source.split('/')[-2:]
(artist_dir,
album_dir) = get_and_make_artist_and_album_dirs(artist, album, target)
print('Calling convert_album', file=sys.stderr)
convert_album(album_dir, target_format, source, bitrate, copy_non_sound,
num_processes)
