def test_format_event_basic(self): processors = [teeth_overlord.service._format_event, _return_event_processor] structlog.configure(processors=processors) log = structlog.wrap_logger(structlog.ReturnLogger()) logged_msg = log.msg("hello {word}", word='world') self.assertEqual(logged_msg, "hello world")
def setUp(self): with open("./tests/data/eq-mwss.json") as fb: data = fb.read() self.response = json.loads(data) self.log = wrap_logger(logging.getLogger(__name__)) self.image_names = ["Image1", "Image2"]
def logger(name=__name__, uuid=False, timestamp=False): """ Configure and return a new logger for hivy modules """ processors = [JSONRenderer()] if uuid: processors.append(add_unique_id) if uuid: processors.append(add_timestamp) return wrap_logger(logbook.Logger(name), processors=processors)
def get_wrapped_logger(): return wrap_logger( logging.getLogger(__name__), processors=[ filter_by_level, add_timestamp, JSONRenderer(separators=(', ', ':'), sort_keys=True) ] )
def test_no_format_keys(self): """Check that we get an exception if you don't provide enough keys to format a log message requiring format """ processors = [teeth_overlord.service._format_event, _return_event_processor] structlog.configure(processors=processors) log = structlog.wrap_logger(structlog.ReturnLogger()) self.assertRaises(KeyError, log.msg, "hello {word}")
def logger(name=__name__): ''' Configure and return a new logger for hivy modules ''' return wrap_logger( logbook.Logger(name), processors=[ add_unique_id, add_timestamp, JSONRenderer(), ] )
def make_log(self, dbfilepath=None): """ Resets and recreates log if one not already created, using optional dbfilepath for file output. """ self.log = wrap_logger(FoveaPrintLogger(dbfilepath=\ os.path.join(self._dirpath,dbfilepath)), wrapper_class=SemanticLogger, processors=[ev_store, KeyValueRenderer()], ) #JSONRenderer(indent=1, sort_keys=True)]) # reference to the database self.db = self.log._logger.db
def setUp(self): with open("./tests/data/eq-mwss.json") as fb: data = fb.read() self.reply = json.loads(data) with open("./tests/data/134.0005.json") as fb: survey_data = fb.read() self.survey = json.loads(survey_data) with open("./tests/data/EDC_134_20170301_1000.csv", "rb") as fb: self.check = fb.read() self.log = wrap_logger(logging.getLogger(__name__))
def main(): """ Push content to the structure logging. """ structlog.configure( processors=[ structlog.stdlib.filter_by_level, structlog.stdlib.add_log_level, structlog.stdlib.PositionalArgumentsFormatter(), # structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S", # structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S", structlog.processors.TimeStamper(fmt="iso", utc=False), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, structlog.processors.JSONRenderer() ], context_class=dict, logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, cache_logger_on_first_use=True, ) print "starting loop" log = structlog.wrap_logger(ROOT_LOGGER) log.info(profile="test.example.1", resource_count=10, resource_max=10) log.info(profile="test.example.2", resource_count=10, resource_max=10) count1 = 10 count2 = 10 for _ in range(10000): if random.randint(0, 1) == 0: count1 -= 1 else: count1 += 1 count1 = max(count1, 0) count1 = min(count1, 10) log.info(profile="test.example.1", resource_count=count1, resource_max=10) if random.randint(0, 1) == 0: count2 -= 1 else: count2 += 1 count2 = max(count2, 0) count2 = min(count2, 10) log.info(profile="test.example.2", resource_count=count2, resource_max=10) seconds = random.randint(0, 30) print "test.example.1", count1, "test.example.2", count2 time.sleep(seconds)
def getLogger(name): return wrap_logger( logging.getLogger(name), processors=[ filter_by_level, add_logger_name, add_caller_info, #local_var_info, unorder_dict, TimeStamper(fmt="ISO", utc=False), format_exc_info, PositionalArgumentsFormatter(), alternate_dev_formatter() ], wrapper_class=BoundLogger, )
def log_factory(handler, level, namespace): """ Opinionated logger factory. """ logger = logging.getLogger(namespace) logger.setLevel(level) if not logger.handlers: logger.addHandler(handler) return structlog.wrap_logger( logger, processors=[ structlog.stdlib.filter_by_level, structlog.stdlib.add_log_level, structlog.stdlib.add_logger_name, structlog.processors.TimeStamper(fmt='iso', utc=True, key='created_at'), structlog.processors.JSONRenderer() ] )
def __init__(self, response, seq_nr=0, log=None): """Create a transformer object to process a survey response.""" self.response = response self.ids = Survey.identifiers(response, seq_nr=seq_nr) if self.ids is None: raise UserWarning("Missing identifiers") if log is None: self.log = wrap_logger(logging.getLogger(__name__)) else: self.log = Survey.bind_logger(log, self.ids) # Enforce that child classes have defn and pattern attributes for attr in ("defn", "pattern"): if not hasattr(self.__class__, attr): raise UserWarning("Missing class attribute: {0}".format(attr)) self.survey = Survey.load_survey(self.ids, self.pattern) self.image_transformer = ImageTransformer(self.log, self.survey, self.response, sequence_no=self.ids.seq_nr, base_image_path=SDX_FTP_IMAGE_PATH)
def __init__(self, name, make_log=False, dbfilepath=None, dirpath='.'): """ By default, does not create a log for diagnostics unless make_log=True. Log can be set with make_log() method after object created. """ self.name = name self.global_count = 0 self._dirpath = dirpath if make_log: self.log = wrap_logger(FoveaPrintLogger(dbfilepath=\ os.path.join(dirpath,dbfilepath)), wrapper_class=SemanticLogger, processors=[ev_store, KeyValueRenderer()], ) #JSONRenderer(indent=1, sort_keys=True)]) # reference to the database self.db = self.log._logger.db else: self.log = None self.db = None # store any metadata associated with log table entries self.log_items_digest = {} self.name_to_digest = {}
from entityservice.logger_setup import setup_structlog celery = Celery('tasks', broker=Config.CELERY_BROKER_URL, backend=Config.CELERY_RESULT_BACKEND ) celery.conf.task_annotations = Config.CELERY_ANNOTATIONS celery.conf.task_acks_late = Config.CELERY_ACKS_LATE celery.conf.task_routes = Config.CELERY_ROUTES celery.conf.broker_transport_options = Config.CELERY_BROKER_TRANSPORT_OPTIONS celery.conf.result_backend_transport_options = Config.CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS celery.conf.worker_prefetch_multiplier = Config.CELERYD_PREFETCH_MULTIPLIER celery.conf.worker_max_tasks_per_child = Config.CELERYD_MAX_TASKS_PER_CHILD # Set up our logging setup_structlog() logger = structlog.wrap_logger(logging.getLogger('entityservice.tasks')) logger.info("Setting up entityservice worker") logger.debug("Debug logging enabled") @task_prerun.connect() def configure_structlog(sender, body=None, **kwargs): task = kwargs['task'] task.logger = logger.new( task_id=kwargs['task_id'], task_name=sender.__name__ )
from distutils.util import strtobool from flask import json, flash, Markup, render_template, redirect, request, url_for from structlog import wrap_logger from frontstage import app from frontstage.common.authorisation import jwt_authorization from frontstage.common.message_helper import from_internal, refine from frontstage.controllers.conversation_controller import get_conversation, get_conversation_list, \ remove_unread_label, send_message, try_message_count_from_session, get_message_count_from_api from frontstage.controllers.survey_controller import get_survey from frontstage.exceptions.exceptions import ApiError from frontstage.models import SecureMessagingForm from frontstage.views.secure_messaging import secure_message_bp logger = wrap_logger(logging.getLogger(__name__)) @secure_message_bp.route('/threads/<thread_id>', methods=['GET', 'POST']) @jwt_authorization(request) def view_conversation(session, thread_id): """Endpoint to view conversations by thread_id""" party_id = session.get_party_id() logger.info("Getting conversation", thread_id=thread_id, party_id=party_id) conversation = get_conversation(thread_id) logger.info('Successfully retrieved conversation', thread_id=thread_id, party_id=party_id) try: refined_conversation = [ refine(message) for message in reversed(conversation['messages'])
from behave import given, when, then from acceptance_tests.features.pages import sign_in_respondent, change_enrolment_status, reporting_unit, surveys_todo from acceptance_tests import browser from logging import getLogger from structlog import wrap_logger logger = wrap_logger(getLogger(__name__)) @given('the internal user is on the ru details page') @when('the internal user is back on the ru details page') def internal_user_view_ru_details(_): reporting_unit.go_to('49900000001') @when('the user clicks either disable or re-enable') def click_enrolment_status_link(_): browser.find_by_id('change-enrolment-status').click() @when('the user clicks the confirm button') def click_confirm_button(_): browser.find_by_id('confirm-btn').click() @then('the user is redirected to a confirmation screen') def confirm_verification_screen(_): assert 'Edit collection exercise details' in browser.title
return event_dict def get_log_renderer(key: str) -> Any: if key == 'json': return structlog.processors.JSONRenderer( sort_keys=True, ensure_ascii=False) elif key == 'console': return structlog.dev.ConsoleRenderer() else: raise ValueError("Unexpected logs renderer " + key) logger = structlog.wrap_logger( structlog.PrintLogger(), processors=[ add_timestamp_logproc, get_log_renderer(config.LOGS_RENDERER), ]) HTML_TAG_RE = re.compile(r"<[^>]+>") def strip_html(html: str) -> str: return HTML_TAG_RE.sub("", html) Entry = namedtuple('Entry', ['title', 'pubdate', 'description', 'link']) def from_feed_entry(e: feedparser.FeedParserDict) -> Entry: return Entry(
import zmq.ssh from tinydb import Query, TinyDB from adaptive_scheduler._scheduler import ( ext, make_job_script, queue, submit_cmd, cancel, ) ctx = zmq.asyncio.Context() logger = logging.getLogger("adaptive_scheduler.server") logger.setLevel(logging.INFO) log = structlog.wrap_logger(logger) class MaxRestartsReached(Exception): """Jobs can fail instantly because of a error in your Python code which results jobs being started indefinitely.""" def _dispatch(request, db_fname): request_type, request_arg = request log.debug("got a request", request=request) try: if request_type == "start": job_id = request_arg # workers send us their slurm ID for us to fill in # give the worker a job and send back the fname to the worker fname = _choose_fname(db_fname, job_id)
eventname = self.event['event'] user = self.event['user'] status = self.event['status'] val = self.event['val'] cur = self.db.cursor() cur.execute("INSERT INTO Events VALUES (%i, '%s', '%s', '%s', %f);" % \ (id, eventname, user, status, val)) self.db.commit() #print("In MyPL.msg: event_list = %s, event = %s" % (self.event_list, # self.event)) err = debug = info = warning = error = critical = log = msg log = wrap_logger(MyPrintLogger(), wrapper_class=SemanticLogger, processors=[ev_store, KeyValueRenderer()], #context_class=structlog.threadlocal.wrap_dict(dict) #dict ) #JSONRenderer(indent=1, sort_keys=True)]) # ---------------------------- log = log.bind(user='******', val=3.1416) log.msg('begin', status='sofarsogood') log.user_error('user.forgot_towel') log.msg('done', status='nope') log = log.bind(user='******', val=-1.01) log.msg('begin', status='yarp') log.user_action('user.remembered_towel') log.msg('done') print("\n")
import calendar import time def timestamper(logger, log_method, event_dict): event_dict["timestamp"] = calendar.timegm(time.gmtime()) return event_dict is_dropped = False def dropper(logger, log_method, event_dict): global is_dropped print("\t@", log_method, event_dict, "@") if is_dropped: raise structlog.DropEvent return event_dict logger = structlog.get_logger() logger = structlog.wrap_logger(logger, processors=[timestamper, dropper]) logger = logger.bind(user="******", some_key=23) logging.basicConfig(level=logging.DEBUG) logger.info("hello") logger.info("hello") is_dropped = True logger.info("hello2") is_dropped = False logger.info("bye")
from entityservice import database as db from entityservice.serialization import generate_scores from entityservice.object_store import connect_to_object_store from entityservice.settings import Config as config from entityservice.utils import fmt_bytes, iterable_to_stream con_app.add_api(pathlib.Path("openapi.yaml"), base_path='/', options={'swagger_ui': False}, strict_validation=config.CONNEXION_STRICT_VALIDATION, validate_responses=config.CONNEXION_RESPONSE_VALIDATION) # Config could be Config, DevelopmentConfig or ProductionConfig app.config.from_object(config) logger = structlog.wrap_logger(app.logger) # Tracer setup (currently just trace all requests) flask_tracer = FlaskTracing(initialize_tracer, True, app) @app.cli.command('initdb') def initdb_command(): """Initializes the database after a short delay.""" db.init_db(5) print('Initialised the database.') @app.before_first_request def before_first_request(): db_min_connections = config.FLASK_DB_MIN_CONNECTIONS db_max_connections = config.FLASK_DB_MAX_CONNECTIONS
from __future__ import absolute_import, division, print_function import structlog class ListLogger(object): def __init__(self): self.events = [] def add_event(self, **kw): self.events.append(kw) logger = ListLogger() log = structlog.wrap_logger( logger, processors=[structlog.processors.TimeStamper(fmt="iso")] ) log.add_event("ford", towel=True) log.add_event("trillian", towel=False) print(logger.events)
event_dict['host'] = os.uname()[1] event_dict['version'] = app.config['VERSION'] if session: event_dict['session_id'] = session.get('session_id') if request: try: event_dict['ip_address'] = request.remote_addr except: event_dict['ip_address'] = 'unknown' return event_dict # Add a handler to write log messages to a file if app.config.get('LOG_FILE'): application_fh = logging.FileHandler(app.config['LOG_APPLICATION']) application_fh.setLevel(app.config['LOG_APPLICATION_LEVEL']) app.logger.addHandler(application_fh) error_fh = logging.FileHandler(app.config['LOG_ERROR']) error_fh.setLevel(app.config['LOG_ERROR_LEVEL']) app.logger.addHandler(error_fh) # Wrap the application logger with structlog to format the output logger = wrap_logger(app.logger, processors=[add_fields, JSONRenderer(indent=None)])
""" Experimenting with using structlog with hope of supporting it in Lightbus """ import logging import sys import structlog def event_dict_ordering(logger, method_name, event_dict): ordered = {"event": event_dict.pop("event")} ordered.update(**event_dict) return ordered structlog.configure(processors=[ event_dict_ordering, structlog.stdlib.add_log_level, structlog.stdlib.add_logger_name, structlog.processors.TimeStamper(fmt="iso"), structlog.dev.ConsoleRenderer() if sys.stdout.isatty() else structlog. processors.JSONRenderer(), ]) if __name__ == "__main__": log = structlog.wrap_logger(logging.getLogger("test")) log.warning("hello from std", foo=1) log.info("Loaded plugins", plugins={...}, context={"service_name": "..."})
from io import StringIO as cio import logging, structlog as sl from threading import Lock import time # log setup: log = logging.getLogger() handler = logging.StreamHandler(cio()) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) log.addHandler(handler) log.setLevel(10) # structlog setup. lets creeate the (threadsafe) Memory logger: class Str: def info(msg, stream=cio(), lock=Lock()): with lock: stream.write(msg) stream.flush() slog = sl.wrap_logger(Str, [lambda _, __, ev: '%(event)s' % ev]) for m in slog.info, log.info: print('structlog:' if m == slog.info else 'stdlib log') t1 = time.time() for i in range(100000): m('info_msg') print(time.time() - t1)
from django.shortcuts import render from django.utils import timezone from django.views.generic import ListView, TemplateView, View from rest_framework.views import APIView from rest_framework.viewsets import ModelViewSet from rest_framework.response import Response from reports.formatmixins import CSVResponseMixin from reports.models import Report, FixityReportItem, FormatReportItem from reports.routines import ReportRunner from reports.serializers import FixityReportSerializer, FormatReportSerializer logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) logger = wrap_logger(logger) class DashboardView(TemplateView): template_name = 'reports/main.html' def get_context_data(self, **kwargs): context = super(DashboardView, self).get_context_data(**kwargs) context['fixity_reports'] = {} context['format_reports'] = {} context['fixity_reports']['objects'] = Report.objects.filter( report_type='fixity') context['fixity_reports']['running'] = Report.objects.filter( report_type='fixity', process_status='started').count() context['fixity_reports']['queued'] = Report.objects.filter( report_type='fixity', process_status='queued').count()
from structlog import PrintLogger, wrap_logger from structlog.processors import JSONRenderer, TimeStamper # renderers must come last log = wrap_logger(PrintLogger(), processors=[TimeStamper(), JSONRenderer()])
from adaptive_scheduler.utils import ( _deserialize, _get_npoints, _serialize, log_exception, maybe_lst, ) ctx = zmq.Context() logger = logging.getLogger("adaptive_scheduler.client") logger.setLevel(logging.INFO) log = structlog.wrap_logger( logger, processors=[ structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False), structlog.processors.JSONRenderer(), ], ) def _add_log_file_handler(log_fname): fh = logging.FileHandler(log_fname) logger.addHandler(fh) def get_learner( url: str, log_fname: str, job_id: str, job_name: str ) -> Tuple[BaseLearner, Union[str, List[str]]]: """Get a learner from the database running at `url` and this learner's
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import logging import sys import structlog # Logger for all queries run or accessed query_run_log = logging.getLogger("flowmachine").getChild("query_run_log") ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.INFO) query_run_log.addHandler(ch) query_run_log = structlog.wrap_logger(query_run_log)
def create_and_wrap_logger(logger_name): logger = wrap_logger(logging.getLogger(logger_name)) logger.info("START", version=__version__) return logger
def get(name=__name__): """Return a structlog.""" return wrap_logger(logging.getLogger('payment.' + name)).bind()
APP.openapi = custom_openapi with open('terms.txt', 'r') as content_file: terms_and_conditions = content_file.read() LOGGER = logging.getLogger() LOGGER.setLevel(logging.INFO) HANDLER = TimedRotatingFileHandler(os.path.join( os.environ["ICEES_API_LOG_PATH"], "server", )) LOGGER.addHandler(HANDLER) LOGGER = wrap_logger(LOGGER, processors=[JSONRenderer()]) @APP.middleware("http") async def fix_tabular_outputs(request: Request, call_next): """Fix tabular outputs.""" response = await call_next(request) timestamp = strftime('%Y-%b-%d %H:%M:%S') LOGGER.info( event="request", timestamp=timestamp, remote_addr=request.client.host, method=request.method, schema=request.url.scheme, full_path=request.url.path,
if session: event_dict['session_id'] = session.get('session_id') if request: try: event_dict['ip_address'] = request.headers['X-Forwarded-For'].split(',')[0].strip() except: event_dict['ip_address'] = 'unknown' return event_dict # Add a handler to write log messages to a file if app.config.get('LOG_FILE'): file_handler = RotatingFileHandler(app.config['LOG_FILENAME'], app.config['LOG_MAXBYTES'], app.config['LOG_BACKUPS'], 'a', encoding='utf-8') file_handler.setLevel(logging.DEBUG) app.logger.addHandler(file_handler) # Wrap the application logger with structlog to format the output logger = wrap_logger( app.logger, processors=[ add_fields, JSONRenderer(indent=None) ] )
def create_app(config_name=None): app = Flask(__name__) app.name = "response_operations_ui" app_config = f'config.{config_name or os.environ.get("APP_SETTINGS", "Config")}' app.config.from_object(app_config) # Load css and js assets assets = Environment(app) if app.config['DEBUG'] or app.config['TESTING']: assets.cache = False assets.manifest = None assets.url = app.static_url_path app.jinja_env.add_extension('jinja2.ext.do') app.register_blueprint(filter_blueprint) app.url_map.strict_slashes = False app.secret_key = app.config['RESPONSE_OPERATIONS_UI_SECRET'] # Zipkin zipkin = Zipkin(app=app, sample_rate=app.config.get("ZIPKIN_SAMPLE_RATE")) requestsdefaulter.default_headers(zipkin.create_http_headers_for_new_span) logger_initial_config(service_name='response-operations-ui', log_level=app.config['LOGGING_LEVEL']) logger = wrap_logger(logging.getLogger(__name__)) logger.info('Logger created', log_level=app.config['LOGGING_LEVEL']) login_manager = LoginManager(app) login_manager.init_app(app) login_manager.login_view = "sign_in_bp.sign_in" @app.context_processor def inject_availability_message(): redis_avail_msg = app.config['SESSION_REDIS'] if len(redis_avail_msg.keys('AVAILABILITY_MESSAGE_RES_OPS')) == 1: return { "availability_message": redis_avail_msg.get('AVAILABILITY_MESSAGE_RES_OPS').decode( 'utf-8') } return {} @login_manager.user_loader def user_loader(user_id): return User(user_id) if cf.detected: with app.app_context(): # If deploying in cloudfoundry set config to use cf redis instance logger.info( 'Cloudfoundry detected, setting service configurations') service = cf.redis app.config['REDIS_HOST'] = service.credentials['host'] app.config['REDIS_PORT'] = service.credentials['port'] # wrap in the flask server side session manager and back it by redis app.config['SESSION_REDIS'] = redis.StrictRedis( host=app.config['REDIS_HOST'], port=app.config['REDIS_PORT'], db=app.config['REDIS_DB']) app.jinja_environment.lstrip_blocks = True if app.config['DEBUG'] or os.environ.get('JINJA_RELOAD'): app.jinja_env.auto_reload = True Session(app) setup_blueprints(app) return app
from os.path import join as pjoin, basename, dirname import shutil import logging import traceback from structlog import wrap_logger from structlog.processors import JSONRenderer import luigi from luigi.local_target import LocalFileSystem from wagl.acquisition import acquisitions from wagl.singlefile_workflow import DataStandardisation from s2pkg.fmask_cophub import prepare_dataset, fmask from s2pkg.package import package ERROR_LOGGER = wrap_logger(logging.getLogger('ard-error'), processors=[JSONRenderer(indent=1, sort_keys=True)]) INTERFACE_LOGGER = logging.getLogger('luigi-interface') @luigi.Task.event_handler(luigi.Event.FAILURE) def on_failure(task, exception): """Capture any Task Failure here.""" ERROR_LOGGER.error(task=task.get_task_family(), params=task.to_str_params(), level1=task.level1, exception=exception.__str__(), traceback=traceback.format_exc().splitlines()) class WorkDir(luigi.Task): """
import os.path import glob import re import sys import subprocess import shlex from pathlib import Path from functools import cmp_to_key from structlog import get_logger, wrap_logger from structlog.dev import ConsoleRenderer import yaml import semver import pkgconfig glog = wrap_logger(get_logger(), processors=[ConsoleRenderer(pad_event=42, colors=True)]) cc = os.getenv('CC', 'gcc') cxx = os.getenv('CXX', 'g++') syslibs = [] libpaths = re.findall(r'SEARCH_DIR\("(.+?)"\);', subprocess.check_output(shlex.split('bash -c "ld --verbose | grep SEARCH_DIR"')).decode('utf-8')) + os.getenv('LD_LIBRARY_PATH', '').split(':') def find_lib(lib, pth): glog.info('find_lib()', lib=lib) for lp in pth: if os.path.isfile(lp + '/' + lib): return lp + '/' + lib glog.failure('unable to find a library', lib=lib) sys.exit(1)
import logging import structlog from flask import Blueprint, jsonify from application.exceptions import RasError log = structlog.wrap_logger(logging.getLogger(__name__)) error_blueprint = Blueprint('error_handlers', __name__) @error_blueprint.app_errorhandler(Exception) def handle_error(error): if isinstance(error, RasError): response = jsonify(error.to_dict()) response.status_code = error.status_code else: response = jsonify({'errors': [str(error)]}) response.status_code = 500 return response
def logging_config(): """ Structured logging configuration setup. With the correct configuration, we can also use this to pipe messages to user via GUI. https://www.structlog.org/en/stable/getting-started.html for more info. """ # configure our *structlog* configuration with a processor chain structlog.configure( context_class=dict, wrapper_class=structlog.stdlib.BoundLogger, processors=[ structlog.processors.TimeStamper(fmt='iso'), # format_exc_info, structlog.processors.StackInfoRenderer(), custom_processors.add_structlog_level, # order_keys, custom_processors.OrderKeys(), # custom processor. orders keys so it is easier to visualize structlog.stdlib.ProcessorFormatter.wrap_for_formatter, ], logger_factory=structlog.stdlib.LoggerFactory(), ) # custom format renderer for the console output stream. Can change colors and styles if desired. stream_formatter = structlog.stdlib.ProcessorFormatter( processor=custom_console_renderer.ConsoleRenderer(), # Below is the default console renderer, as defined by the structlog devs. # # Note: Had issues displaying and logging exception info. # Took source and revised a custom version of it. # Old configuration commented out for reference. # structlog.dev.ConsoleRenderer( # # level_styles= # # { # # 'info:': '\033[31m', # # # colorama ANSI sequences # # # print(structlog.dev.ConsoleRenderer.get_default_level_styles()) for further example # # } # ), # foreign_pre_chain= # [ # # structlog.processors.TimeStamper(fmt='iso'), # # custom_processors.OrderKeys(keys=['timestamp', 'level', 'event', 'msg', 'path', 'exc_info']), # # format_exc_info, # # custom_processors.add_structlog_level, # # structlog.stdlib.add_log_level, # # order_keys, # # OrderKeys(keys=['timestamp', 'level', 'event', 'msg', 'exc_info']), # structlog.stdlib.ProcessorFormatter.wrap_for_formatter, # # ], # keep_exc_info=True, # keep_stack_info=True, ) # custom formatter for the log file file_formatter = structlog.stdlib.ProcessorFormatter( processor=structlog.processors.JSONRenderer(sort_keys=False), foreign_pre_chain= [ structlog.processors.TimeStamper(fmt='iso'), structlog.stdlib.add_log_level, custom_processors.OrderKeys() # custom_processors.OrderKeys(keys=['timestamp', 'level', 'event', 'msg', 'path', 'exc_info']) ], ) # file handler. responsible for writing stuff to log file_handler = logging.FileHandler('samplify.log') # set formatter for file handler. responsible for formatting the log file (example: json output) file_handler.setFormatter(file_formatter) # stream handler. responsible for piping stuff to console. stream_handler = logging.StreamHandler() # set formatter for the stream handler. responsible for formatting the output stream (example: colors) stream_handler.setFormatter(stream_formatter) # create a standard logger from python module root_logger = logging.getLogger() # apply handlers to the standard logger root_logger.addHandler(stream_handler) root_logger.addHandler(file_handler) # set a global log level root_logger.setLevel(logging.DEBUG) # set log level for stdout/stderr stream stream_handler.setLevel(logging.INFO) # wrap the standard logger from python lib for structlog lib use. structlog.wrap_logger( root_logger, processors=[ structlog.processors.TimeStamper(fmt='iso'), # custom_processors.add_structlog_level, structlog.stdlib.add_log_level, ] )
def get_wrapped_logger(logger_name: str = 'root', **kwargs): """ Returns a struct log equivalent for the named logger """ return structlog.wrap_logger(logging.getLogger(logger_name), COMMON_PROCESSORS, **kwargs)
def __init__(self, logger=None): self.logger = logger or wrap_logger(logging.getLogger(__name__)) self._ftp = SDXFTP(self.logger, settings.FTP_HOST, settings.FTP_USER, settings.FTP_PASS)
def create_app(config_name=None): csp_policy = copy.deepcopy(CSP_POLICY) app = Flask(__name__) app_config = f'config.{config_name or os.environ.get("APP_SETTINGS", "Config")}' app.config.from_object(app_config) if Config.WTF_CSRF_ENABLED: Talisman( app, content_security_policy=csp_policy, content_security_policy_nonce_in=['script-src'], force_https=False, # this is handled at the load balancer legacy_content_security_policy_header=True, strict_transport_security=True, strict_transport_security_max_age=ONE_YEAR_IN_SECONDS, referrer_policy=DEFAULT_REFERRER_POLICY, frame_options='SAMEORIGIN', frame_options_allow_from=None, session_cookie_secure=True, session_cookie_http_only=True) app.name = "response_operations_ui" CSRFProtect(app) # Load css and js assets assets = Environment(app) if app.config['DEBUG'] or app.config['TESTING']: assets.cache = False assets.manifest = None if not app.config['DEBUG']: app.wsgi_app = GCPLoadBalancer(app.wsgi_app) assets.url = app.static_url_path app.jinja_env.add_extension('jinja2.ext.do') app.register_blueprint(filter_blueprint) app.url_map.strict_slashes = False app.secret_key = app.config['RESPONSE_OPERATIONS_UI_SECRET'] logger_initial_config(service_name='response-operations-ui', log_level=app.config['LOGGING_LEVEL']) logger = wrap_logger(logging.getLogger(__name__)) logger.info('Logger created', log_level=app.config['LOGGING_LEVEL']) login_manager = LoginManager(app) login_manager.init_app(app) login_manager.login_view = "sign_in_bp.sign_in" @app.context_processor def inject_availability_message(): redis_avail_msg = app.config['SESSION_REDIS'] if len(redis_avail_msg.keys('AVAILABILITY_MESSAGE_RES_OPS')) == 1: return { "availability_message": redis_avail_msg.get('AVAILABILITY_MESSAGE_RES_OPS').decode('utf-8') } return {} @login_manager.user_loader def user_loader(user_id): username = session.get('username') return User(user_id, username) # wrap in the flask server side session manager and back it by redis app.config['SESSION_REDIS'] = redis.StrictRedis(host=app.config['REDIS_HOST'], port=app.config['REDIS_PORT'], db=app.config['REDIS_DB']) app.jinja_environment.lstrip_blocks = True if app.config['DEBUG'] or os.environ.get('JINJA_RELOAD'): app.jinja_env.auto_reload = True Session(app) setup_blueprints(app) return app
from flask_security import RoleMixin from flask_security import SQLAlchemySessionUserDatastore from flask_security.forms import PasswordField from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer from sqlalchemy import String, TIMESTAMP from sqlalchemy.dialects.postgresql import JSONB, UUID from sqlalchemy.exc import IntegrityError, SQLAlchemyError from sqlalchemy.orm import relationship, backref from sqlalchemy.sql.expression import func from structlog import wrap_logger from wtforms import HiddenField from console import db, app, settings from console.helpers.exceptions import UserExistsError, UserCreationError logger = wrap_logger(logging.getLogger(__name__)) class RolesUsers(db.Model): __tablename__ = 'roles_users' id = Column(Integer(), primary_key=True) user_id = Column('user_id', Integer(), ForeignKey('flaskuser.id')) role_id = Column('role_id', Integer(), ForeignKey('role.id')) class Role(db.Model, RoleMixin): __tablename__ = 'role' id = Column(Integer(), primary_key=True) name = Column(String(80), unique=True) description = Column(String(255))
import logging from structlog import wrap_logger from structlog import PrintLogger logger = wrap_logger(logging.getLogger("iac-stub")) class Iac_Stub: def __init__(self): self.iac_fixed_response = None def get_iac_stub(self, IacEntered): """Create the fixed iac response as a Python dictionary object. This can then be converted to a json object later on.""" PrintLogger().info("Now in the get_iac_stub function") iac_fixed_response = { 'caseId': '0337c579-ce9d-4357-a620-5e4c565cfac1', 'caseRef': '1000000000000002', 'iac': IacEntered, 'active': True, 'questionSet': None, 'lastUsedDateTime': 1543502892050 } PrintLogger().info("Now returning the iac fixed response: " + repr(iac_fixed_response)) return iac_fixed_response