def test_format_event_basic(self):
processors = [teeth_overlord.service._format_event,
_return_event_processor]
structlog.configure(processors=processors)
log = structlog.wrap_logger(structlog.ReturnLogger())
logged_msg = log.msg("hello {word}", word='world')
self.assertEqual(logged_msg, "hello world")
def setUp(self):
with open("./tests/data/eq-mwss.json") as fb:
data = fb.read()
self.response = json.loads(data)
self.log = wrap_logger(logging.getLogger(__name__))
self.image_names = ["Image1", "Image2"]
def logger(name=__name__, uuid=False, timestamp=False):
""" Configure and return a new logger for hivy modules """
processors = [JSONRenderer()]
if uuid:
processors.append(add_unique_id)
if uuid:
processors.append(add_timestamp)
return wrap_logger(logbook.Logger(name), processors=processors)
def get_wrapped_logger():
return wrap_logger(
logging.getLogger(__name__),
processors=[
filter_by_level,
add_timestamp,
JSONRenderer(separators=(', ', ':'), sort_keys=True)
]
)
def test_no_format_keys(self):
"""Check that we get an exception if you don't provide enough keys to
format a log message requiring format
"""
processors = [teeth_overlord.service._format_event,
_return_event_processor]
structlog.configure(processors=processors)
log = structlog.wrap_logger(structlog.ReturnLogger())
self.assertRaises(KeyError, log.msg, "hello {word}")
def logger(name=__name__):
''' Configure and return a new logger for hivy modules '''
return wrap_logger(
logbook.Logger(name),
processors=[
add_unique_id,
add_timestamp,
JSONRenderer(),
]
)
def make_log(self, dbfilepath=None):
"""
Resets and recreates log if one not already created, using
optional dbfilepath for file output.
"""
self.log = wrap_logger(FoveaPrintLogger(dbfilepath=\
os.path.join(self._dirpath,dbfilepath)),
wrapper_class=SemanticLogger,
processors=[ev_store, KeyValueRenderer()],
) #JSONRenderer(indent=1, sort_keys=True)])
# reference to the database
self.db = self.log._logger.db
def setUp(self):
with open("./tests/data/eq-mwss.json") as fb:
data = fb.read()
self.reply = json.loads(data)
with open("./tests/data/134.0005.json") as fb:
survey_data = fb.read()
self.survey = json.loads(survey_data)
with open("./tests/data/EDC_134_20170301_1000.csv", "rb") as fb:
self.check = fb.read()
self.log = wrap_logger(logging.getLogger(__name__))
def main():
""" Push content to the structure logging. """
structlog.configure(
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
# structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S",
# structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S",
structlog.processors.TimeStamper(fmt="iso", utc=False),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.JSONRenderer()
],
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
print "starting loop"
log = structlog.wrap_logger(ROOT_LOGGER)
log.info(profile="test.example.1", resource_count=10, resource_max=10)
log.info(profile="test.example.2", resource_count=10, resource_max=10)
count1 = 10
count2 = 10
for _ in range(10000):
if random.randint(0, 1) == 0:
count1 -= 1
else:
count1 += 1
count1 = max(count1, 0)
count1 = min(count1, 10)
log.info(profile="test.example.1", resource_count=count1,
resource_max=10)
if random.randint(0, 1) == 0:
count2 -= 1
else:
count2 += 1
count2 = max(count2, 0)
count2 = min(count2, 10)
log.info(profile="test.example.2", resource_count=count2,
resource_max=10)
seconds = random.randint(0, 30)
print "test.example.1", count1, "test.example.2", count2
time.sleep(seconds)
def getLogger(name):
return wrap_logger(
logging.getLogger(name),
processors=[
filter_by_level,
add_logger_name,
add_caller_info,
#local_var_info,
unorder_dict,
TimeStamper(fmt="ISO", utc=False),
format_exc_info,
PositionalArgumentsFormatter(),
alternate_dev_formatter()
],
wrapper_class=BoundLogger,
)
def log_factory(handler, level, namespace):
""" Opinionated logger factory. """
logger = logging.getLogger(namespace)
logger.setLevel(level)
if not logger.handlers:
logger.addHandler(handler)
return structlog.wrap_logger(
logger,
processors=[
structlog.stdlib.filter_by_level,
structlog.stdlib.add_log_level,
structlog.stdlib.add_logger_name,
structlog.processors.TimeStamper(fmt='iso', utc=True, key='created_at'),
structlog.processors.JSONRenderer()
]
)
def __init__(self, response, seq_nr=0, log=None):
"""Create a transformer object to process a survey response."""
self.response = response
self.ids = Survey.identifiers(response, seq_nr=seq_nr)
if self.ids is None:
raise UserWarning("Missing identifiers")
if log is None:
self.log = wrap_logger(logging.getLogger(__name__))
else:
self.log = Survey.bind_logger(log, self.ids)
# Enforce that child classes have defn and pattern attributes
for attr in ("defn", "pattern"):
if not hasattr(self.__class__, attr):
raise UserWarning("Missing class attribute: {0}".format(attr))
self.survey = Survey.load_survey(self.ids, self.pattern)
self.image_transformer = ImageTransformer(self.log, self.survey, self.response,
sequence_no=self.ids.seq_nr, base_image_path=SDX_FTP_IMAGE_PATH)
def __init__(self, name, make_log=False, dbfilepath=None, dirpath='.'):
"""
By default, does not create a log for diagnostics unless make_log=True.
Log can be set with make_log() method after object created.
"""
self.name = name
self.global_count = 0
self._dirpath = dirpath
if make_log:
self.log = wrap_logger(FoveaPrintLogger(dbfilepath=\
os.path.join(dirpath,dbfilepath)),
wrapper_class=SemanticLogger,
processors=[ev_store, KeyValueRenderer()],
) #JSONRenderer(indent=1, sort_keys=True)])
# reference to the database
self.db = self.log._logger.db
else:
self.log = None
self.db = None
# store any metadata associated with log table entries
self.log_items_digest = {}
self.name_to_digest = {}
from entityservice.logger_setup import setup_structlog
celery = Celery('tasks',
broker=Config.CELERY_BROKER_URL,
backend=Config.CELERY_RESULT_BACKEND
)
celery.conf.task_annotations = Config.CELERY_ANNOTATIONS
celery.conf.task_acks_late = Config.CELERY_ACKS_LATE
celery.conf.task_routes = Config.CELERY_ROUTES
celery.conf.broker_transport_options = Config.CELERY_BROKER_TRANSPORT_OPTIONS
celery.conf.result_backend_transport_options = Config.CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS
celery.conf.worker_prefetch_multiplier = Config.CELERYD_PREFETCH_MULTIPLIER
celery.conf.worker_max_tasks_per_child = Config.CELERYD_MAX_TASKS_PER_CHILD
# Set up our logging
setup_structlog()
logger = structlog.wrap_logger(logging.getLogger('entityservice.tasks'))
logger.info("Setting up entityservice worker")
logger.debug("Debug logging enabled")
@task_prerun.connect()
def configure_structlog(sender, body=None, **kwargs):
task = kwargs['task']
task.logger = logger.new(
task_id=kwargs['task_id'],
task_name=sender.__name__
)
from distutils.util import strtobool
from flask import json, flash, Markup, render_template, redirect, request, url_for
from structlog import wrap_logger
from frontstage import app
from frontstage.common.authorisation import jwt_authorization
from frontstage.common.message_helper import from_internal, refine
from frontstage.controllers.conversation_controller import get_conversation, get_conversation_list, \
remove_unread_label, send_message, try_message_count_from_session, get_message_count_from_api
from frontstage.controllers.survey_controller import get_survey
from frontstage.exceptions.exceptions import ApiError
from frontstage.models import SecureMessagingForm
from frontstage.views.secure_messaging import secure_message_bp
logger = wrap_logger(logging.getLogger(__name__))
@secure_message_bp.route('/threads/<thread_id>', methods=['GET', 'POST'])
@jwt_authorization(request)
def view_conversation(session, thread_id):
"""Endpoint to view conversations by thread_id"""
party_id = session.get_party_id()
logger.info("Getting conversation", thread_id=thread_id, party_id=party_id)
conversation = get_conversation(thread_id)
logger.info('Successfully retrieved conversation',
thread_id=thread_id,
party_id=party_id)
try:
refined_conversation = [
refine(message) for message in reversed(conversation['messages'])
from behave import given, when, then
from acceptance_tests.features.pages import sign_in_respondent, change_enrolment_status, reporting_unit, surveys_todo
from acceptance_tests import browser
from logging import getLogger
from structlog import wrap_logger
logger = wrap_logger(getLogger(__name__))
@given('the internal user is on the ru details page')
@when('the internal user is back on the ru details page')
def internal_user_view_ru_details(_):
reporting_unit.go_to('49900000001')
@when('the user clicks either disable or re-enable')
def click_enrolment_status_link(_):
browser.find_by_id('change-enrolment-status').click()
@when('the user clicks the confirm button')
def click_confirm_button(_):
browser.find_by_id('confirm-btn').click()
@then('the user is redirected to a confirmation screen')
def confirm_verification_screen(_):
assert 'Edit collection exercise details' in browser.title
return event_dict
def get_log_renderer(key: str) -> Any:
if key == 'json':
return structlog.processors.JSONRenderer(
sort_keys=True, ensure_ascii=False)
elif key == 'console':
return structlog.dev.ConsoleRenderer()
else:
raise ValueError("Unexpected logs renderer " + key)
logger = structlog.wrap_logger(
structlog.PrintLogger(),
processors=[
add_timestamp_logproc,
get_log_renderer(config.LOGS_RENDERER),
])
HTML_TAG_RE = re.compile(r"<[^>]+>")
def strip_html(html: str) -> str:
return HTML_TAG_RE.sub("", html)
Entry = namedtuple('Entry', ['title', 'pubdate', 'description', 'link'])
def from_feed_entry(e: feedparser.FeedParserDict) -> Entry:
return Entry(
import zmq.ssh
from tinydb import Query, TinyDB
from adaptive_scheduler._scheduler import (
ext,
make_job_script,
queue,
submit_cmd,
cancel,
)
ctx = zmq.asyncio.Context()
logger = logging.getLogger("adaptive_scheduler.server")
logger.setLevel(logging.INFO)
log = structlog.wrap_logger(logger)
class MaxRestartsReached(Exception):
"""Jobs can fail instantly because of a error in
your Python code which results jobs being started indefinitely."""
def _dispatch(request, db_fname):
request_type, request_arg = request
log.debug("got a request", request=request)
try:
if request_type == "start":
job_id = request_arg # workers send us their slurm ID for us to fill in
# give the worker a job and send back the fname to the worker
fname = _choose_fname(db_fname, job_id)
eventname = self.event['event']
user = self.event['user']
status = self.event['status']
val = self.event['val']
cur = self.db.cursor()
cur.execute("INSERT INTO Events VALUES (%i, '%s', '%s', '%s', %f);" % \
(id, eventname, user, status, val))
self.db.commit()
#print("In MyPL.msg: event_list = %s, event = %s" % (self.event_list,
# self.event))
err = debug = info = warning = error = critical = log = msg
log = wrap_logger(MyPrintLogger(), wrapper_class=SemanticLogger,
processors=[ev_store, KeyValueRenderer()],
#context_class=structlog.threadlocal.wrap_dict(dict) #dict
) #JSONRenderer(indent=1, sort_keys=True)])
# ----------------------------
log = log.bind(user='******', val=3.1416)
log.msg('begin', status='sofarsogood')
log.user_error('user.forgot_towel')
log.msg('done', status='nope')
log = log.bind(user='******', val=-1.01)
log.msg('begin', status='yarp')
log.user_action('user.remembered_towel')
log.msg('done')
print("\n")
import calendar
import time
def timestamper(logger, log_method, event_dict):
event_dict["timestamp"] = calendar.timegm(time.gmtime())
return event_dict
is_dropped = False
def dropper(logger, log_method, event_dict):
global is_dropped
print("\t@", log_method, event_dict, "@")
if is_dropped:
raise structlog.DropEvent
return event_dict
logger = structlog.get_logger()
logger = structlog.wrap_logger(logger, processors=[timestamper, dropper])
logger = logger.bind(user="******", some_key=23)
logging.basicConfig(level=logging.DEBUG)
logger.info("hello")
logger.info("hello")
is_dropped = True
logger.info("hello2")
is_dropped = False
logger.info("bye")
from entityservice import database as db
from entityservice.serialization import generate_scores
from entityservice.object_store import connect_to_object_store
from entityservice.settings import Config as config
from entityservice.utils import fmt_bytes, iterable_to_stream
con_app.add_api(pathlib.Path("openapi.yaml"),
base_path='/',
options={'swagger_ui': False},
strict_validation=config.CONNEXION_STRICT_VALIDATION,
validate_responses=config.CONNEXION_RESPONSE_VALIDATION)
# Config could be Config, DevelopmentConfig or ProductionConfig
app.config.from_object(config)
logger = structlog.wrap_logger(app.logger)
# Tracer setup (currently just trace all requests)
flask_tracer = FlaskTracing(initialize_tracer, True, app)
@app.cli.command('initdb')
def initdb_command():
"""Initializes the database after a short delay."""
db.init_db(5)
print('Initialised the database.')
@app.before_first_request
def before_first_request():
db_min_connections = config.FLASK_DB_MIN_CONNECTIONS
db_max_connections = config.FLASK_DB_MAX_CONNECTIONS
from __future__ import absolute_import, division, print_function
import structlog
class ListLogger(object):
def __init__(self):
self.events = []
def add_event(self, **kw):
self.events.append(kw)
logger = ListLogger()
log = structlog.wrap_logger(
logger,
processors=[structlog.processors.TimeStamper(fmt="iso")]
)
log.add_event("ford", towel=True)
log.add_event("trillian", towel=False)
print(logger.events)
event_dict['host'] = os.uname()[1]
event_dict['version'] = app.config['VERSION']
if session:
event_dict['session_id'] = session.get('session_id')
if request:
try:
event_dict['ip_address'] = request.remote_addr
except:
event_dict['ip_address'] = 'unknown'
return event_dict
# Add a handler to write log messages to a file
if app.config.get('LOG_FILE'):
application_fh = logging.FileHandler(app.config['LOG_APPLICATION'])
application_fh.setLevel(app.config['LOG_APPLICATION_LEVEL'])
app.logger.addHandler(application_fh)
error_fh = logging.FileHandler(app.config['LOG_ERROR'])
error_fh.setLevel(app.config['LOG_ERROR_LEVEL'])
app.logger.addHandler(error_fh)
# Wrap the application logger with structlog to format the output
logger = wrap_logger(app.logger,
processors=[add_fields,
JSONRenderer(indent=None)])
""" Experimenting with using structlog with hope of supporting it in Lightbus
"""
import logging
import sys
import structlog
def event_dict_ordering(logger, method_name, event_dict):
ordered = {"event": event_dict.pop("event")}
ordered.update(**event_dict)
return ordered
structlog.configure(processors=[
event_dict_ordering,
structlog.stdlib.add_log_level,
structlog.stdlib.add_logger_name,
structlog.processors.TimeStamper(fmt="iso"),
structlog.dev.ConsoleRenderer() if sys.stdout.isatty() else structlog.
processors.JSONRenderer(),
])
if __name__ == "__main__":
log = structlog.wrap_logger(logging.getLogger("test"))
log.warning("hello from std", foo=1)
log.info("Loaded plugins", plugins={...}, context={"service_name": "..."})
from io import StringIO as cio
import logging, structlog as sl
from threading import Lock
import time
# log setup:
log = logging.getLogger()
handler = logging.StreamHandler(cio())
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
log.addHandler(handler)
log.setLevel(10)
# structlog setup. lets creeate the (threadsafe) Memory logger:
class Str:
def info(msg, stream=cio(), lock=Lock()):
with lock:
stream.write(msg)
stream.flush()
slog = sl.wrap_logger(Str, [lambda _, __, ev: '%(event)s' % ev])
for m in slog.info, log.info:
print('structlog:' if m == slog.info else 'stdlib log')
t1 = time.time()
for i in range(100000):
m('info_msg')
print(time.time() - t1)
from django.shortcuts import render
from django.utils import timezone
from django.views.generic import ListView, TemplateView, View
from rest_framework.views import APIView
from rest_framework.viewsets import ModelViewSet
from rest_framework.response import Response
from reports.formatmixins import CSVResponseMixin
from reports.models import Report, FixityReportItem, FormatReportItem
from reports.routines import ReportRunner
from reports.serializers import FixityReportSerializer, FormatReportSerializer
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger = wrap_logger(logger)
class DashboardView(TemplateView):
template_name = 'reports/main.html'
def get_context_data(self, **kwargs):
context = super(DashboardView, self).get_context_data(**kwargs)
context['fixity_reports'] = {}
context['format_reports'] = {}
context['fixity_reports']['objects'] = Report.objects.filter(
report_type='fixity')
context['fixity_reports']['running'] = Report.objects.filter(
report_type='fixity', process_status='started').count()
context['fixity_reports']['queued'] = Report.objects.filter(
report_type='fixity', process_status='queued').count()
from structlog import PrintLogger, wrap_logger from structlog.processors import JSONRenderer, TimeStamper # renderers must come last log = wrap_logger(PrintLogger(), processors=[TimeStamper(), JSONRenderer()])
from adaptive_scheduler.utils import (
_deserialize,
_get_npoints,
_serialize,
log_exception,
maybe_lst,
)
ctx = zmq.Context()
logger = logging.getLogger("adaptive_scheduler.client")
logger.setLevel(logging.INFO)
log = structlog.wrap_logger(
logger,
processors=[
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False),
structlog.processors.JSONRenderer(),
],
)
def _add_log_file_handler(log_fname):
fh = logging.FileHandler(log_fname)
logger.addHandler(fh)
def get_learner(
url: str, log_fname: str, job_id: str, job_name: str
) -> Tuple[BaseLearner, Union[str, List[str]]]:
"""Get a learner from the database running at `url` and this learner's
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import logging
import sys
import structlog
# Logger for all queries run or accessed
query_run_log = logging.getLogger("flowmachine").getChild("query_run_log")
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
query_run_log.addHandler(ch)
query_run_log = structlog.wrap_logger(query_run_log)
def create_and_wrap_logger(logger_name):
logger = wrap_logger(logging.getLogger(logger_name))
logger.info("START", version=__version__)
return logger
def get(name=__name__):
"""Return a structlog."""
return wrap_logger(logging.getLogger('payment.' + name)).bind()
APP.openapi = custom_openapi
with open('terms.txt', 'r') as content_file:
terms_and_conditions = content_file.read()
LOGGER = logging.getLogger()
LOGGER.setLevel(logging.INFO)
HANDLER = TimedRotatingFileHandler(os.path.join(
os.environ["ICEES_API_LOG_PATH"],
"server",
))
LOGGER.addHandler(HANDLER)
LOGGER = wrap_logger(LOGGER, processors=[JSONRenderer()])
@APP.middleware("http")
async def fix_tabular_outputs(request: Request, call_next):
"""Fix tabular outputs."""
response = await call_next(request)
timestamp = strftime('%Y-%b-%d %H:%M:%S')
LOGGER.info(
event="request",
timestamp=timestamp,
remote_addr=request.client.host,
method=request.method,
schema=request.url.scheme,
full_path=request.url.path,
if session:
event_dict['session_id'] = session.get('session_id')
if request:
try:
event_dict['ip_address'] = request.headers['X-Forwarded-For'].split(',')[0].strip()
except:
event_dict['ip_address'] = 'unknown'
return event_dict
# Add a handler to write log messages to a file
if app.config.get('LOG_FILE'):
file_handler = RotatingFileHandler(app.config['LOG_FILENAME'],
app.config['LOG_MAXBYTES'],
app.config['LOG_BACKUPS'],
'a',
encoding='utf-8')
file_handler.setLevel(logging.DEBUG)
app.logger.addHandler(file_handler)
# Wrap the application logger with structlog to format the output
logger = wrap_logger(
app.logger,
processors=[
add_fields,
JSONRenderer(indent=None)
]
)
def create_app(config_name=None):
app = Flask(__name__)
app.name = "response_operations_ui"
app_config = f'config.{config_name or os.environ.get("APP_SETTINGS", "Config")}'
app.config.from_object(app_config)
# Load css and js assets
assets = Environment(app)
if app.config['DEBUG'] or app.config['TESTING']:
assets.cache = False
assets.manifest = None
assets.url = app.static_url_path
app.jinja_env.add_extension('jinja2.ext.do')
app.register_blueprint(filter_blueprint)
app.url_map.strict_slashes = False
app.secret_key = app.config['RESPONSE_OPERATIONS_UI_SECRET']
# Zipkin
zipkin = Zipkin(app=app, sample_rate=app.config.get("ZIPKIN_SAMPLE_RATE"))
requestsdefaulter.default_headers(zipkin.create_http_headers_for_new_span)
logger_initial_config(service_name='response-operations-ui',
log_level=app.config['LOGGING_LEVEL'])
logger = wrap_logger(logging.getLogger(__name__))
logger.info('Logger created', log_level=app.config['LOGGING_LEVEL'])
login_manager = LoginManager(app)
login_manager.init_app(app)
login_manager.login_view = "sign_in_bp.sign_in"
@app.context_processor
def inject_availability_message():
redis_avail_msg = app.config['SESSION_REDIS']
if len(redis_avail_msg.keys('AVAILABILITY_MESSAGE_RES_OPS')) == 1:
return {
"availability_message":
redis_avail_msg.get('AVAILABILITY_MESSAGE_RES_OPS').decode(
'utf-8')
}
return {}
@login_manager.user_loader
def user_loader(user_id):
return User(user_id)
if cf.detected:
with app.app_context():
# If deploying in cloudfoundry set config to use cf redis instance
logger.info(
'Cloudfoundry detected, setting service configurations')
service = cf.redis
app.config['REDIS_HOST'] = service.credentials['host']
app.config['REDIS_PORT'] = service.credentials['port']
# wrap in the flask server side session manager and back it by redis
app.config['SESSION_REDIS'] = redis.StrictRedis(
host=app.config['REDIS_HOST'],
port=app.config['REDIS_PORT'],
db=app.config['REDIS_DB'])
app.jinja_environment.lstrip_blocks = True
if app.config['DEBUG'] or os.environ.get('JINJA_RELOAD'):
app.jinja_env.auto_reload = True
Session(app)
setup_blueprints(app)
return app
from os.path import join as pjoin, basename, dirname
import shutil
import logging
import traceback
from structlog import wrap_logger
from structlog.processors import JSONRenderer
import luigi
from luigi.local_target import LocalFileSystem
from wagl.acquisition import acquisitions
from wagl.singlefile_workflow import DataStandardisation
from s2pkg.fmask_cophub import prepare_dataset, fmask
from s2pkg.package import package
ERROR_LOGGER = wrap_logger(logging.getLogger('ard-error'),
processors=[JSONRenderer(indent=1, sort_keys=True)])
INTERFACE_LOGGER = logging.getLogger('luigi-interface')
@luigi.Task.event_handler(luigi.Event.FAILURE)
def on_failure(task, exception):
"""Capture any Task Failure here."""
ERROR_LOGGER.error(task=task.get_task_family(),
params=task.to_str_params(),
level1=task.level1,
exception=exception.__str__(),
traceback=traceback.format_exc().splitlines())
class WorkDir(luigi.Task):
"""
import os.path
import glob
import re
import sys
import subprocess
import shlex
from pathlib import Path
from functools import cmp_to_key
from structlog import get_logger, wrap_logger
from structlog.dev import ConsoleRenderer
import yaml
import semver
import pkgconfig
glog = wrap_logger(get_logger(), processors=[ConsoleRenderer(pad_event=42, colors=True)])
cc = os.getenv('CC', 'gcc')
cxx = os.getenv('CXX', 'g++')
syslibs = []
libpaths = re.findall(r'SEARCH_DIR\("(.+?)"\);', subprocess.check_output(shlex.split('bash -c "ld --verbose | grep SEARCH_DIR"')).decode('utf-8')) + os.getenv('LD_LIBRARY_PATH', '').split(':')
def find_lib(lib, pth):
glog.info('find_lib()', lib=lib)
for lp in pth:
if os.path.isfile(lp + '/' + lib):
return lp + '/' + lib
glog.failure('unable to find a library', lib=lib)
sys.exit(1)
import logging
import structlog
from flask import Blueprint, jsonify
from application.exceptions import RasError
log = structlog.wrap_logger(logging.getLogger(__name__))
error_blueprint = Blueprint('error_handlers', __name__)
@error_blueprint.app_errorhandler(Exception)
def handle_error(error):
if isinstance(error, RasError):
response = jsonify(error.to_dict())
response.status_code = error.status_code
else:
response = jsonify({'errors': [str(error)]})
response.status_code = 500
return response
def logging_config():
"""
Structured logging configuration setup. With the correct configuration, we can also use this to
pipe messages to user via GUI.
https://www.structlog.org/en/stable/getting-started.html for more info.
"""
# configure our *structlog* configuration with a processor chain
structlog.configure(
context_class=dict,
wrapper_class=structlog.stdlib.BoundLogger,
processors=[
structlog.processors.TimeStamper(fmt='iso'),
# format_exc_info,
structlog.processors.StackInfoRenderer(),
custom_processors.add_structlog_level,
# order_keys,
custom_processors.OrderKeys(), # custom processor. orders keys so it is easier to visualize
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
logger_factory=structlog.stdlib.LoggerFactory(),
)
# custom format renderer for the console output stream. Can change colors and styles if desired.
stream_formatter = structlog.stdlib.ProcessorFormatter(
processor=custom_console_renderer.ConsoleRenderer(),
# Below is the default console renderer, as defined by the structlog devs.
#
# Note: Had issues displaying and logging exception info.
# Took source and revised a custom version of it.
# Old configuration commented out for reference.
# structlog.dev.ConsoleRenderer(
# # level_styles=
# # {
# # 'info:': '\033[31m',
# # # colorama ANSI sequences
# # # print(structlog.dev.ConsoleRenderer.get_default_level_styles()) for further example
# # }
# ),
# foreign_pre_chain=
# [
# # structlog.processors.TimeStamper(fmt='iso'),
# # custom_processors.OrderKeys(keys=['timestamp', 'level', 'event', 'msg', 'path', 'exc_info']),
# # format_exc_info,
# # custom_processors.add_structlog_level,
# # structlog.stdlib.add_log_level,
# # order_keys,
# # OrderKeys(keys=['timestamp', 'level', 'event', 'msg', 'exc_info']),
# structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
#
# ],
# keep_exc_info=True,
# keep_stack_info=True,
)
# custom formatter for the log file
file_formatter = structlog.stdlib.ProcessorFormatter(
processor=structlog.processors.JSONRenderer(sort_keys=False),
foreign_pre_chain=
[
structlog.processors.TimeStamper(fmt='iso'),
structlog.stdlib.add_log_level,
custom_processors.OrderKeys()
# custom_processors.OrderKeys(keys=['timestamp', 'level', 'event', 'msg', 'path', 'exc_info'])
],
)
# file handler. responsible for writing stuff to log
file_handler = logging.FileHandler('samplify.log')
# set formatter for file handler. responsible for formatting the log file (example: json output)
file_handler.setFormatter(file_formatter)
# stream handler. responsible for piping stuff to console.
stream_handler = logging.StreamHandler()
# set formatter for the stream handler. responsible for formatting the output stream (example: colors)
stream_handler.setFormatter(stream_formatter)
# create a standard logger from python module
root_logger = logging.getLogger()
# apply handlers to the standard logger
root_logger.addHandler(stream_handler)
root_logger.addHandler(file_handler)
# set a global log level
root_logger.setLevel(logging.DEBUG)
# set log level for stdout/stderr stream
stream_handler.setLevel(logging.INFO)
# wrap the standard logger from python lib for structlog lib use.
structlog.wrap_logger(
root_logger,
processors=[
structlog.processors.TimeStamper(fmt='iso'),
# custom_processors.add_structlog_level,
structlog.stdlib.add_log_level,
]
)
def get_wrapped_logger(logger_name: str = 'root', **kwargs):
""" Returns a struct log equivalent for the named logger """
return structlog.wrap_logger(logging.getLogger(logger_name),
COMMON_PROCESSORS, **kwargs)
def __init__(self, logger=None):
self.logger = logger or wrap_logger(logging.getLogger(__name__))
self._ftp = SDXFTP(self.logger, settings.FTP_HOST, settings.FTP_USER,
settings.FTP_PASS)
def create_app(config_name=None):
csp_policy = copy.deepcopy(CSP_POLICY)
app = Flask(__name__)
app_config = f'config.{config_name or os.environ.get("APP_SETTINGS", "Config")}'
app.config.from_object(app_config)
if Config.WTF_CSRF_ENABLED:
Talisman(
app,
content_security_policy=csp_policy,
content_security_policy_nonce_in=['script-src'],
force_https=False, # this is handled at the load balancer
legacy_content_security_policy_header=True,
strict_transport_security=True,
strict_transport_security_max_age=ONE_YEAR_IN_SECONDS,
referrer_policy=DEFAULT_REFERRER_POLICY,
frame_options='SAMEORIGIN',
frame_options_allow_from=None,
session_cookie_secure=True,
session_cookie_http_only=True)
app.name = "response_operations_ui"
CSRFProtect(app)
# Load css and js assets
assets = Environment(app)
if app.config['DEBUG'] or app.config['TESTING']:
assets.cache = False
assets.manifest = None
if not app.config['DEBUG']:
app.wsgi_app = GCPLoadBalancer(app.wsgi_app)
assets.url = app.static_url_path
app.jinja_env.add_extension('jinja2.ext.do')
app.register_blueprint(filter_blueprint)
app.url_map.strict_slashes = False
app.secret_key = app.config['RESPONSE_OPERATIONS_UI_SECRET']
logger_initial_config(service_name='response-operations-ui', log_level=app.config['LOGGING_LEVEL'])
logger = wrap_logger(logging.getLogger(__name__))
logger.info('Logger created', log_level=app.config['LOGGING_LEVEL'])
login_manager = LoginManager(app)
login_manager.init_app(app)
login_manager.login_view = "sign_in_bp.sign_in"
@app.context_processor
def inject_availability_message():
redis_avail_msg = app.config['SESSION_REDIS']
if len(redis_avail_msg.keys('AVAILABILITY_MESSAGE_RES_OPS')) == 1:
return {
"availability_message": redis_avail_msg.get('AVAILABILITY_MESSAGE_RES_OPS').decode('utf-8')
}
return {}
@login_manager.user_loader
def user_loader(user_id):
username = session.get('username')
return User(user_id, username)
# wrap in the flask server side session manager and back it by redis
app.config['SESSION_REDIS'] = redis.StrictRedis(host=app.config['REDIS_HOST'],
port=app.config['REDIS_PORT'],
db=app.config['REDIS_DB'])
app.jinja_environment.lstrip_blocks = True
if app.config['DEBUG'] or os.environ.get('JINJA_RELOAD'):
app.jinja_env.auto_reload = True
Session(app)
setup_blueprints(app)
return app
from flask_security import RoleMixin
from flask_security import SQLAlchemySessionUserDatastore
from flask_security.forms import PasswordField
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer
from sqlalchemy import String, TIMESTAMP
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
from sqlalchemy.orm import relationship, backref
from sqlalchemy.sql.expression import func
from structlog import wrap_logger
from wtforms import HiddenField
from console import db, app, settings
from console.helpers.exceptions import UserExistsError, UserCreationError
logger = wrap_logger(logging.getLogger(__name__))
class RolesUsers(db.Model):
__tablename__ = 'roles_users'
id = Column(Integer(), primary_key=True)
user_id = Column('user_id', Integer(), ForeignKey('flaskuser.id'))
role_id = Column('role_id', Integer(), ForeignKey('role.id'))
class Role(db.Model, RoleMixin):
__tablename__ = 'role'
id = Column(Integer(), primary_key=True)
name = Column(String(80), unique=True)
description = Column(String(255))
import logging
from structlog import wrap_logger
from structlog import PrintLogger
logger = wrap_logger(logging.getLogger("iac-stub"))
class Iac_Stub:
def __init__(self):
self.iac_fixed_response = None
def get_iac_stub(self, IacEntered):
"""Create the fixed iac response as a Python dictionary object. This can then be converted to a json object later on."""
PrintLogger().info("Now in the get_iac_stub function")
iac_fixed_response = {
'caseId': '0337c579-ce9d-4357-a620-5e4c565cfac1',
'caseRef': '1000000000000002',
'iac': IacEntered,
'active': True,
'questionSet': None,
'lastUsedDateTime': 1543502892050
}
PrintLogger().info("Now returning the iac fixed response: " +
repr(iac_fixed_response))
return iac_fixed_response