Beispiel #1
0
 def test_format_event_basic(self):
     processors = [teeth_overlord.service._format_event,
                   _return_event_processor]
     structlog.configure(processors=processors)
     log = structlog.wrap_logger(structlog.ReturnLogger())
     logged_msg = log.msg("hello {word}", word='world')
     self.assertEqual(logged_msg, "hello world")
    def setUp(self):
        with open("./tests/data/eq-mwss.json") as fb:
            data = fb.read()
        self.response = json.loads(data)

        self.log = wrap_logger(logging.getLogger(__name__))
        self.image_names = ["Image1", "Image2"]
Beispiel #3
0
def logger(name=__name__, uuid=False, timestamp=False):
    """ Configure and return a new logger for hivy modules """
    processors = [JSONRenderer()]
    if uuid:
        processors.append(add_unique_id)
    if uuid:
        processors.append(add_timestamp)
    return wrap_logger(logbook.Logger(name), processors=processors)
Beispiel #4
0
def get_wrapped_logger():
    return wrap_logger(
        logging.getLogger(__name__),
        processors=[
            filter_by_level,
            add_timestamp,
            JSONRenderer(separators=(', ', ':'), sort_keys=True)
        ]
    )
Beispiel #5
0
 def test_no_format_keys(self):
     """Check that we get an exception if you don't provide enough keys to
     format a log message requiring format
     """
     processors = [teeth_overlord.service._format_event,
                   _return_event_processor]
     structlog.configure(processors=processors)
     log = structlog.wrap_logger(structlog.ReturnLogger())
     self.assertRaises(KeyError, log.msg, "hello {word}")
Beispiel #6
0
def logger(name=__name__):
    ''' Configure and return a new logger for hivy modules '''
    return wrap_logger(
        logbook.Logger(name),
        processors=[
            add_unique_id,
            add_timestamp,
            JSONRenderer(),
        ]
    )
Beispiel #7
0
 def make_log(self, dbfilepath=None):
     """
     Resets and recreates log if one not already created, using
     optional dbfilepath for file output.
     """
     self.log = wrap_logger(FoveaPrintLogger(dbfilepath=\
                                    os.path.join(self._dirpath,dbfilepath)),
                            wrapper_class=SemanticLogger,
                            processors=[ev_store, KeyValueRenderer()],
                            ) #JSONRenderer(indent=1, sort_keys=True)])
     # reference to the database
     self.db = self.log._logger.db
    def setUp(self):
        with open("./tests/data/eq-mwss.json") as fb:
            data = fb.read()
        self.reply = json.loads(data)

        with open("./tests/data/134.0005.json") as fb:
            survey_data = fb.read()
        self.survey = json.loads(survey_data)

        with open("./tests/data/EDC_134_20170301_1000.csv", "rb") as fb:
            self.check = fb.read()
        self.log = wrap_logger(logging.getLogger(__name__))
Beispiel #9
0
def main():
    """ Push content to the structure logging. """

    structlog.configure(
        processors=[
            structlog.stdlib.filter_by_level,
            structlog.stdlib.add_log_level,
            structlog.stdlib.PositionalArgumentsFormatter(),
            # structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S",
            # structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S",
            structlog.processors.TimeStamper(fmt="iso", utc=False),
            structlog.processors.StackInfoRenderer(),
            structlog.processors.format_exc_info,
            structlog.processors.JSONRenderer()
        ],
        context_class=dict,
        logger_factory=structlog.stdlib.LoggerFactory(),
        wrapper_class=structlog.stdlib.BoundLogger,
        cache_logger_on_first_use=True,
    )
    print "starting loop"

    log = structlog.wrap_logger(ROOT_LOGGER)
    log.info(profile="test.example.1", resource_count=10, resource_max=10)
    log.info(profile="test.example.2", resource_count=10, resource_max=10)

    count1 = 10
    count2 = 10
    for _ in range(10000):
        if random.randint(0, 1) == 0:
            count1 -= 1
        else:
            count1 += 1
        count1 = max(count1, 0)
        count1 = min(count1, 10)
        log.info(profile="test.example.1", resource_count=count1,
                 resource_max=10)

        if random.randint(0, 1) == 0:
            count2 -= 1
        else:
            count2 += 1

        count2 = max(count2, 0)
        count2 = min(count2, 10)

        log.info(profile="test.example.2", resource_count=count2,
                 resource_max=10)
        seconds = random.randint(0, 30)
        print "test.example.1", count1, "test.example.2", count2
        time.sleep(seconds)
Beispiel #10
0
def getLogger(name):
    return wrap_logger(
        logging.getLogger(name),
        processors=[
            filter_by_level,
            add_logger_name,
            add_caller_info,
            #local_var_info,
            unorder_dict,
            TimeStamper(fmt="ISO", utc=False),
            format_exc_info,
            PositionalArgumentsFormatter(),
            alternate_dev_formatter()
        ],
        wrapper_class=BoundLogger,
    )
Beispiel #11
0
def log_factory(handler, level, namespace):
    """ Opinionated logger factory. """
    logger = logging.getLogger(namespace)
    logger.setLevel(level)

    if not logger.handlers:
        logger.addHandler(handler)

    return structlog.wrap_logger(
        logger,
        processors=[
            structlog.stdlib.filter_by_level,
            structlog.stdlib.add_log_level,
            structlog.stdlib.add_logger_name,
            structlog.processors.TimeStamper(fmt='iso', utc=True, key='created_at'),
            structlog.processors.JSONRenderer()
        ]
    )
    def __init__(self, response, seq_nr=0, log=None):
        """Create a transformer object to process a survey response."""
        self.response = response
        self.ids = Survey.identifiers(response, seq_nr=seq_nr)

        if self.ids is None:
            raise UserWarning("Missing identifiers")

        if log is None:
            self.log = wrap_logger(logging.getLogger(__name__))
        else:
            self.log = Survey.bind_logger(log, self.ids)

        # Enforce that child classes have defn and pattern attributes
        for attr in ("defn", "pattern"):
            if not hasattr(self.__class__, attr):
                raise UserWarning("Missing class attribute: {0}".format(attr))

        self.survey = Survey.load_survey(self.ids, self.pattern)
        self.image_transformer = ImageTransformer(self.log, self.survey, self.response,
                                                  sequence_no=self.ids.seq_nr, base_image_path=SDX_FTP_IMAGE_PATH)
Beispiel #13
0
 def __init__(self, name, make_log=False, dbfilepath=None, dirpath='.'):
     """
     By default, does not create a log for diagnostics unless make_log=True.
     Log can be set with make_log() method after object created.
     """
     self.name = name
     self.global_count = 0
     self._dirpath = dirpath
     if make_log:
         self.log = wrap_logger(FoveaPrintLogger(dbfilepath=\
                                          os.path.join(dirpath,dbfilepath)),
                                wrapper_class=SemanticLogger,
                                processors=[ev_store, KeyValueRenderer()],
                    ) #JSONRenderer(indent=1, sort_keys=True)])
         # reference to the database
         self.db = self.log._logger.db
     else:
         self.log = None
         self.db = None
     # store any metadata associated with log table entries
     self.log_items_digest = {}
     self.name_to_digest = {}
from entityservice.logger_setup import setup_structlog

celery = Celery('tasks',
                broker=Config.CELERY_BROKER_URL,
                backend=Config.CELERY_RESULT_BACKEND
                )

celery.conf.task_annotations = Config.CELERY_ANNOTATIONS
celery.conf.task_acks_late = Config.CELERY_ACKS_LATE
celery.conf.task_routes = Config.CELERY_ROUTES
celery.conf.broker_transport_options = Config.CELERY_BROKER_TRANSPORT_OPTIONS
celery.conf.result_backend_transport_options = Config.CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS
celery.conf.worker_prefetch_multiplier = Config.CELERYD_PREFETCH_MULTIPLIER
celery.conf.worker_max_tasks_per_child = Config.CELERYD_MAX_TASKS_PER_CHILD


# Set up our logging
setup_structlog()
logger = structlog.wrap_logger(logging.getLogger('entityservice.tasks'))
logger.info("Setting up entityservice worker")
logger.debug("Debug logging enabled")


@task_prerun.connect()
def configure_structlog(sender, body=None, **kwargs):
    task = kwargs['task']
    task.logger = logger.new(
        task_id=kwargs['task_id'],
        task_name=sender.__name__
    )
from distutils.util import strtobool

from flask import json, flash, Markup, render_template, redirect, request, url_for
from structlog import wrap_logger

from frontstage import app
from frontstage.common.authorisation import jwt_authorization
from frontstage.common.message_helper import from_internal, refine
from frontstage.controllers.conversation_controller import get_conversation, get_conversation_list, \
    remove_unread_label, send_message, try_message_count_from_session, get_message_count_from_api
from frontstage.controllers.survey_controller import get_survey
from frontstage.exceptions.exceptions import ApiError
from frontstage.models import SecureMessagingForm
from frontstage.views.secure_messaging import secure_message_bp

logger = wrap_logger(logging.getLogger(__name__))


@secure_message_bp.route('/threads/<thread_id>', methods=['GET', 'POST'])
@jwt_authorization(request)
def view_conversation(session, thread_id):
    """Endpoint to view conversations by thread_id"""
    party_id = session.get_party_id()
    logger.info("Getting conversation", thread_id=thread_id, party_id=party_id)
    conversation = get_conversation(thread_id)
    logger.info('Successfully retrieved conversation',
                thread_id=thread_id,
                party_id=party_id)
    try:
        refined_conversation = [
            refine(message) for message in reversed(conversation['messages'])
Beispiel #16
0
from behave import given, when, then

from acceptance_tests.features.pages import sign_in_respondent, change_enrolment_status, reporting_unit, surveys_todo
from acceptance_tests import browser


from logging import getLogger
from structlog import wrap_logger
logger = wrap_logger(getLogger(__name__))


@given('the internal user is on the ru details page')
@when('the internal user is back on the ru details page')
def internal_user_view_ru_details(_):
    reporting_unit.go_to('49900000001')


@when('the user clicks either disable or re-enable')
def click_enrolment_status_link(_):
    browser.find_by_id('change-enrolment-status').click()


@when('the user clicks the confirm button')
def click_confirm_button(_):
    browser.find_by_id('confirm-btn').click()


@then('the user is redirected to a confirmation screen')
def confirm_verification_screen(_):
    assert 'Edit collection exercise details' in browser.title
Beispiel #17
0
    return event_dict


def get_log_renderer(key: str) -> Any:
    if key == 'json':
        return structlog.processors.JSONRenderer(
            sort_keys=True, ensure_ascii=False)
    elif key == 'console':
        return structlog.dev.ConsoleRenderer()
    else:
        raise ValueError("Unexpected logs renderer " + key)


logger = structlog.wrap_logger(
    structlog.PrintLogger(),
    processors=[
        add_timestamp_logproc,
        get_log_renderer(config.LOGS_RENDERER),
    ])

HTML_TAG_RE = re.compile(r"<[^>]+>")


def strip_html(html: str) -> str:
    return HTML_TAG_RE.sub("", html)


Entry = namedtuple('Entry', ['title', 'pubdate', 'description', 'link'])


def from_feed_entry(e: feedparser.FeedParserDict) -> Entry:
    return Entry(
Beispiel #18
0
import zmq.ssh
from tinydb import Query, TinyDB

from adaptive_scheduler._scheduler import (
    ext,
    make_job_script,
    queue,
    submit_cmd,
    cancel,
)

ctx = zmq.asyncio.Context()

logger = logging.getLogger("adaptive_scheduler.server")
logger.setLevel(logging.INFO)
log = structlog.wrap_logger(logger)


class MaxRestartsReached(Exception):
    """Jobs can fail instantly because of a error in
    your Python code which results jobs being started indefinitely."""


def _dispatch(request, db_fname):
    request_type, request_arg = request
    log.debug("got a request", request=request)
    try:
        if request_type == "start":
            job_id = request_arg  # workers send us their slurm ID for us to fill in
            # give the worker a job and send back the fname to the worker
            fname = _choose_fname(db_fname, job_id)
Beispiel #19
0
        eventname = self.event['event']
        user = self.event['user']
        status = self.event['status']
        val = self.event['val']
        cur = self.db.cursor()
        cur.execute("INSERT INTO Events VALUES (%i, '%s', '%s', '%s', %f);" % \
                    (id, eventname, user, status, val))
        self.db.commit()
        #print("In MyPL.msg: event_list = %s, event = %s" % (self.event_list,
        #                                                     self.event))

    err = debug = info = warning = error = critical = log = msg


log = wrap_logger(MyPrintLogger(), wrapper_class=SemanticLogger,
                  processors=[ev_store, KeyValueRenderer()],
                  #context_class=structlog.threadlocal.wrap_dict(dict) #dict
                  ) #JSONRenderer(indent=1, sort_keys=True)])

# ----------------------------

log = log.bind(user='******', val=3.1416)
log.msg('begin', status='sofarsogood')
log.user_error('user.forgot_towel')
log.msg('done', status='nope')

log = log.bind(user='******', val=-1.01)
log.msg('begin', status='yarp')
log.user_action('user.remembered_towel')
log.msg('done')

print("\n")
import calendar
import time


def timestamper(logger, log_method, event_dict):
    event_dict["timestamp"] = calendar.timegm(time.gmtime())
    return event_dict


is_dropped = False


def dropper(logger, log_method, event_dict):
    global is_dropped
    print("\t@", log_method, event_dict, "@")
    if is_dropped:
        raise structlog.DropEvent
    return event_dict

logger = structlog.get_logger()
logger = structlog.wrap_logger(logger, processors=[timestamper, dropper])
logger = logger.bind(user="******", some_key=23)

logging.basicConfig(level=logging.DEBUG)
logger.info("hello")
logger.info("hello")
is_dropped = True
logger.info("hello2")
is_dropped = False
logger.info("bye")
from entityservice import database as db
from entityservice.serialization import generate_scores
from entityservice.object_store import connect_to_object_store
from entityservice.settings import Config as config
from entityservice.utils import fmt_bytes, iterable_to_stream

con_app.add_api(pathlib.Path("openapi.yaml"),
                base_path='/',
                options={'swagger_ui': False},
                strict_validation=config.CONNEXION_STRICT_VALIDATION,
                validate_responses=config.CONNEXION_RESPONSE_VALIDATION)

# Config could be Config, DevelopmentConfig or ProductionConfig
app.config.from_object(config)

logger = structlog.wrap_logger(app.logger)
# Tracer setup (currently just trace all requests)
flask_tracer = FlaskTracing(initialize_tracer, True, app)


@app.cli.command('initdb')
def initdb_command():
    """Initializes the database after a short delay."""
    db.init_db(5)
    print('Initialised the database.')


@app.before_first_request
def before_first_request():
    db_min_connections = config.FLASK_DB_MIN_CONNECTIONS
    db_max_connections = config.FLASK_DB_MAX_CONNECTIONS
Beispiel #22
0
from __future__ import absolute_import, division, print_function

import structlog


class ListLogger(object):
    def __init__(self):
        self.events = []

    def add_event(self, **kw):
        self.events.append(kw)


logger = ListLogger()

log = structlog.wrap_logger(
    logger,
    processors=[structlog.processors.TimeStamper(fmt="iso")]
)

log.add_event("ford", towel=True)
log.add_event("trillian", towel=False)

print(logger.events)
    event_dict['host'] = os.uname()[1]
    event_dict['version'] = app.config['VERSION']

    if session:
        event_dict['session_id'] = session.get('session_id')

    if request:
        try:

            event_dict['ip_address'] = request.remote_addr
        except:
            event_dict['ip_address'] = 'unknown'

    return event_dict


# Add a handler to write log messages to a file
if app.config.get('LOG_FILE'):
    application_fh = logging.FileHandler(app.config['LOG_APPLICATION'])
    application_fh.setLevel(app.config['LOG_APPLICATION_LEVEL'])
    app.logger.addHandler(application_fh)

    error_fh = logging.FileHandler(app.config['LOG_ERROR'])
    error_fh.setLevel(app.config['LOG_ERROR_LEVEL'])
    app.logger.addHandler(error_fh)

# Wrap the application logger with structlog to format the output
logger = wrap_logger(app.logger,
                     processors=[add_fields,
                                 JSONRenderer(indent=None)])
Beispiel #24
0
""" Experimenting with using structlog with hope of supporting it in Lightbus
"""
import logging
import sys

import structlog


def event_dict_ordering(logger, method_name, event_dict):
    ordered = {"event": event_dict.pop("event")}
    ordered.update(**event_dict)
    return ordered


structlog.configure(processors=[
    event_dict_ordering,
    structlog.stdlib.add_log_level,
    structlog.stdlib.add_logger_name,
    structlog.processors.TimeStamper(fmt="iso"),
    structlog.dev.ConsoleRenderer() if sys.stdout.isatty() else structlog.
    processors.JSONRenderer(),
])

if __name__ == "__main__":
    log = structlog.wrap_logger(logging.getLogger("test"))
    log.warning("hello from std", foo=1)

    log.info("Loaded plugins", plugins={...}, context={"service_name": "..."})
Beispiel #25
0
from io import StringIO as cio
import logging, structlog as sl
from threading import Lock
import time

# log setup:
log = logging.getLogger()
handler = logging.StreamHandler(cio())
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
log.addHandler(handler)
log.setLevel(10)

# structlog setup. lets creeate the (threadsafe) Memory logger:
class Str:
    def info(msg, stream=cio(), lock=Lock()):
        with lock:
            stream.write(msg)
            stream.flush()
slog = sl.wrap_logger(Str, [lambda _, __, ev: '%(event)s' % ev])

for m in slog.info, log.info:
    print('structlog:' if m == slog.info else 'stdlib log')
    t1 = time.time()
    for i in range(100000):
        m('info_msg')
    print(time.time() - t1)

Beispiel #26
0
from django.shortcuts import render
from django.utils import timezone
from django.views.generic import ListView, TemplateView, View

from rest_framework.views import APIView
from rest_framework.viewsets import ModelViewSet
from rest_framework.response import Response

from reports.formatmixins import CSVResponseMixin
from reports.models import Report, FixityReportItem, FormatReportItem
from reports.routines import ReportRunner
from reports.serializers import FixityReportSerializer, FormatReportSerializer

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger = wrap_logger(logger)


class DashboardView(TemplateView):
    template_name = 'reports/main.html'

    def get_context_data(self, **kwargs):
        context = super(DashboardView, self).get_context_data(**kwargs)
        context['fixity_reports'] = {}
        context['format_reports'] = {}
        context['fixity_reports']['objects'] = Report.objects.filter(
            report_type='fixity')
        context['fixity_reports']['running'] = Report.objects.filter(
            report_type='fixity', process_status='started').count()
        context['fixity_reports']['queued'] = Report.objects.filter(
            report_type='fixity', process_status='queued').count()
Beispiel #27
0
from structlog import PrintLogger, wrap_logger
from structlog.processors import JSONRenderer, TimeStamper

# renderers must come last
log = wrap_logger(PrintLogger(), processors=[TimeStamper(), JSONRenderer()])
from adaptive_scheduler.utils import (
    _deserialize,
    _get_npoints,
    _serialize,
    log_exception,
    maybe_lst,
)

ctx = zmq.Context()
logger = logging.getLogger("adaptive_scheduler.client")
logger.setLevel(logging.INFO)
log = structlog.wrap_logger(
    logger,
    processors=[
        structlog.processors.StackInfoRenderer(),
        structlog.processors.format_exc_info,
        structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False),
        structlog.processors.JSONRenderer(),
    ],
)


def _add_log_file_handler(log_fname):
    fh = logging.FileHandler(log_fname)
    logger.addHandler(fh)


def get_learner(
    url: str, log_fname: str, job_id: str, job_name: str
) -> Tuple[BaseLearner, Union[str, List[str]]]:
    """Get a learner from the database running at `url` and this learner's
Beispiel #29
0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import logging
import sys

import structlog

# Logger for all queries run or accessed
query_run_log = logging.getLogger("flowmachine").getChild("query_run_log")
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
query_run_log.addHandler(ch)
query_run_log = structlog.wrap_logger(query_run_log)
Beispiel #30
0
def create_and_wrap_logger(logger_name):
    logger = wrap_logger(logging.getLogger(logger_name))
    logger.info("START", version=__version__)
    return logger
def get(name=__name__):
    """Return a structlog."""
    return wrap_logger(logging.getLogger('payment.' + name)).bind()
Beispiel #32
0
APP.openapi = custom_openapi

with open('terms.txt', 'r') as content_file:
    terms_and_conditions = content_file.read()

LOGGER = logging.getLogger()
LOGGER.setLevel(logging.INFO)

HANDLER = TimedRotatingFileHandler(os.path.join(
    os.environ["ICEES_API_LOG_PATH"],
    "server",
))

LOGGER.addHandler(HANDLER)
LOGGER = wrap_logger(LOGGER, processors=[JSONRenderer()])


@APP.middleware("http")
async def fix_tabular_outputs(request: Request, call_next):
    """Fix tabular outputs."""
    response = await call_next(request)

    timestamp = strftime('%Y-%b-%d %H:%M:%S')
    LOGGER.info(
        event="request",
        timestamp=timestamp,
        remote_addr=request.client.host,
        method=request.method,
        schema=request.url.scheme,
        full_path=request.url.path,
    if session:
        event_dict['session_id'] = session.get('session_id')

    if request:
        try:
            event_dict['ip_address'] = request.headers['X-Forwarded-For'].split(',')[0].strip()
        except:
            event_dict['ip_address'] = 'unknown'

    return event_dict

# Add a handler to write log messages to a file
if app.config.get('LOG_FILE'):
    file_handler = RotatingFileHandler(app.config['LOG_FILENAME'],
                                       app.config['LOG_MAXBYTES'],
                                       app.config['LOG_BACKUPS'],
                                       'a',
                                       encoding='utf-8')
    file_handler.setLevel(logging.DEBUG)
    app.logger.addHandler(file_handler)

# Wrap the application logger with structlog to format the output
logger = wrap_logger(
    app.logger,
    processors=[
        add_fields,
        JSONRenderer(indent=None)
    ]
)
Beispiel #34
0
def create_app(config_name=None):
    app = Flask(__name__)
    app.name = "response_operations_ui"

    app_config = f'config.{config_name or os.environ.get("APP_SETTINGS", "Config")}'
    app.config.from_object(app_config)

    # Load css and js assets
    assets = Environment(app)

    if app.config['DEBUG'] or app.config['TESTING']:
        assets.cache = False
        assets.manifest = None

    assets.url = app.static_url_path

    app.jinja_env.add_extension('jinja2.ext.do')

    app.register_blueprint(filter_blueprint)

    app.url_map.strict_slashes = False
    app.secret_key = app.config['RESPONSE_OPERATIONS_UI_SECRET']

    # Zipkin
    zipkin = Zipkin(app=app, sample_rate=app.config.get("ZIPKIN_SAMPLE_RATE"))
    requestsdefaulter.default_headers(zipkin.create_http_headers_for_new_span)

    logger_initial_config(service_name='response-operations-ui',
                          log_level=app.config['LOGGING_LEVEL'])
    logger = wrap_logger(logging.getLogger(__name__))
    logger.info('Logger created', log_level=app.config['LOGGING_LEVEL'])

    login_manager = LoginManager(app)
    login_manager.init_app(app)
    login_manager.login_view = "sign_in_bp.sign_in"

    @app.context_processor
    def inject_availability_message():

        redis_avail_msg = app.config['SESSION_REDIS']

        if len(redis_avail_msg.keys('AVAILABILITY_MESSAGE_RES_OPS')) == 1:
            return {
                "availability_message":
                redis_avail_msg.get('AVAILABILITY_MESSAGE_RES_OPS').decode(
                    'utf-8')
            }
        return {}

    @login_manager.user_loader
    def user_loader(user_id):
        return User(user_id)

    if cf.detected:
        with app.app_context():
            # If deploying in cloudfoundry set config to use cf redis instance
            logger.info(
                'Cloudfoundry detected, setting service configurations')
            service = cf.redis
            app.config['REDIS_HOST'] = service.credentials['host']
            app.config['REDIS_PORT'] = service.credentials['port']

    # wrap in the flask server side session manager and back it by redis
    app.config['SESSION_REDIS'] = redis.StrictRedis(
        host=app.config['REDIS_HOST'],
        port=app.config['REDIS_PORT'],
        db=app.config['REDIS_DB'])

    app.jinja_environment.lstrip_blocks = True

    if app.config['DEBUG'] or os.environ.get('JINJA_RELOAD'):
        app.jinja_env.auto_reload = True

    Session(app)

    setup_blueprints(app)

    return app
from os.path import join as pjoin, basename, dirname
import shutil
import logging
import traceback
from structlog import wrap_logger
from structlog.processors import JSONRenderer
import luigi
from luigi.local_target import LocalFileSystem

from wagl.acquisition import acquisitions
from wagl.singlefile_workflow import DataStandardisation
from s2pkg.fmask_cophub import prepare_dataset, fmask
from s2pkg.package import package

ERROR_LOGGER = wrap_logger(logging.getLogger('ard-error'),
                           processors=[JSONRenderer(indent=1, sort_keys=True)])
INTERFACE_LOGGER = logging.getLogger('luigi-interface')


@luigi.Task.event_handler(luigi.Event.FAILURE)
def on_failure(task, exception):
    """Capture any Task Failure here."""
    ERROR_LOGGER.error(task=task.get_task_family(),
                       params=task.to_str_params(),
                       level1=task.level1,
                       exception=exception.__str__(),
                       traceback=traceback.format_exc().splitlines())


class WorkDir(luigi.Task):
    """
Beispiel #36
0
import os.path
import glob
import re
import sys
import subprocess
import shlex
from pathlib import Path
from functools import cmp_to_key

from structlog import get_logger, wrap_logger
from structlog.dev import ConsoleRenderer
import yaml
import semver
import pkgconfig

glog = wrap_logger(get_logger(), processors=[ConsoleRenderer(pad_event=42, colors=True)])

cc = os.getenv('CC', 'gcc')
cxx = os.getenv('CXX', 'g++')

syslibs = []
libpaths = re.findall(r'SEARCH_DIR\("(.+?)"\);', subprocess.check_output(shlex.split('bash -c "ld --verbose | grep SEARCH_DIR"')).decode('utf-8')) + os.getenv('LD_LIBRARY_PATH', '').split(':')

def find_lib(lib, pth):
  glog.info('find_lib()', lib=lib)
  for lp in pth:
    if os.path.isfile(lp + '/' + lib):
      return lp + '/' + lib
  glog.failure('unable to find a library', lib=lib)
  sys.exit(1)
  
Beispiel #37
0
import logging

import structlog
from flask import Blueprint, jsonify

from application.exceptions import RasError

log = structlog.wrap_logger(logging.getLogger(__name__))

error_blueprint = Blueprint('error_handlers', __name__)


@error_blueprint.app_errorhandler(Exception)
def handle_error(error):
    if isinstance(error, RasError):
        response = jsonify(error.to_dict())
        response.status_code = error.status_code
    else:
        response = jsonify({'errors': [str(error)]})
        response.status_code = 500
    return response
Beispiel #38
0
    def logging_config():
        """
        Structured logging configuration setup. With the correct configuration, we can also use this to
        pipe messages to user via GUI.

        https://www.structlog.org/en/stable/getting-started.html for more info.
        """

        # configure our *structlog* configuration with a processor chain
        structlog.configure(
            context_class=dict,
            wrapper_class=structlog.stdlib.BoundLogger,
            processors=[
                structlog.processors.TimeStamper(fmt='iso'),
                # format_exc_info,
                structlog.processors.StackInfoRenderer(),
                custom_processors.add_structlog_level,
                # order_keys,
                custom_processors.OrderKeys(),  # custom processor. orders keys so it is easier to visualize
                structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
            ],
            logger_factory=structlog.stdlib.LoggerFactory(),
        )

        # custom format renderer for the console output stream. Can change colors and styles if desired.
        stream_formatter = structlog.stdlib.ProcessorFormatter(
            processor=custom_console_renderer.ConsoleRenderer(),

            # Below is the default console renderer, as defined by the structlog devs.
            #
            #   Note: Had issues displaying and logging exception info.
            #       Took source and revised a custom version of it.
            #       Old configuration commented out for reference.

            # structlog.dev.ConsoleRenderer(
            # #     level_styles=
            # # {
            # #     'info:': '\033[31m',
            # #     # colorama ANSI sequences
            # #     # print(structlog.dev.ConsoleRenderer.get_default_level_styles()) for further example
            # # }
            # ),
            # foreign_pre_chain=
            # [
            #     # structlog.processors.TimeStamper(fmt='iso'),
            #     # custom_processors.OrderKeys(keys=['timestamp', 'level', 'event', 'msg', 'path', 'exc_info']),
            #     # format_exc_info,
            #     # custom_processors.add_structlog_level,
            #     # structlog.stdlib.add_log_level,
            #     # order_keys,
            #     # OrderKeys(keys=['timestamp', 'level', 'event', 'msg', 'exc_info']),
            #     structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
            #
            # ],
            # keep_exc_info=True,
            # keep_stack_info=True,
        )

        # custom formatter for the log file
        file_formatter = structlog.stdlib.ProcessorFormatter(
            processor=structlog.processors.JSONRenderer(sort_keys=False),
            foreign_pre_chain=
            [
                structlog.processors.TimeStamper(fmt='iso'),
                structlog.stdlib.add_log_level,
                custom_processors.OrderKeys()
                # custom_processors.OrderKeys(keys=['timestamp', 'level', 'event', 'msg', 'path', 'exc_info'])
            ],
        )

        # file handler. responsible for writing stuff to log
        file_handler = logging.FileHandler('samplify.log')

        # set formatter for file handler. responsible for formatting the log file (example: json output)
        file_handler.setFormatter(file_formatter)

        # stream handler. responsible for piping stuff to console.
        stream_handler = logging.StreamHandler()

        # set formatter for the stream handler. responsible for formatting the output stream (example: colors)
        stream_handler.setFormatter(stream_formatter)

        # create a standard logger from python module
        root_logger = logging.getLogger()

        # apply handlers to the standard logger
        root_logger.addHandler(stream_handler)
        root_logger.addHandler(file_handler)

        # set a global log level
        root_logger.setLevel(logging.DEBUG)

        # set log level for stdout/stderr stream
        stream_handler.setLevel(logging.INFO)

        # wrap the standard logger from python lib for structlog lib use.
        structlog.wrap_logger(
            root_logger,
            processors=[
                structlog.processors.TimeStamper(fmt='iso'),
                # custom_processors.add_structlog_level,
                structlog.stdlib.add_log_level,
            ]
        )
Beispiel #39
0
def get_wrapped_logger(logger_name: str = 'root', **kwargs):
    """ Returns a struct log equivalent for the named logger """
    return structlog.wrap_logger(logging.getLogger(logger_name),
                                 COMMON_PROCESSORS, **kwargs)
 def __init__(self, logger=None):
     self.logger = logger or wrap_logger(logging.getLogger(__name__))
     self._ftp = SDXFTP(self.logger, settings.FTP_HOST, settings.FTP_USER,
                        settings.FTP_PASS)
def create_app(config_name=None):
    csp_policy = copy.deepcopy(CSP_POLICY)
    app = Flask(__name__)

    app_config = f'config.{config_name or os.environ.get("APP_SETTINGS", "Config")}'
    app.config.from_object(app_config)

    if Config.WTF_CSRF_ENABLED:
        Talisman(
            app,
            content_security_policy=csp_policy,
            content_security_policy_nonce_in=['script-src'],
            force_https=False,  # this is handled at the load balancer
            legacy_content_security_policy_header=True,
            strict_transport_security=True,
            strict_transport_security_max_age=ONE_YEAR_IN_SECONDS,
            referrer_policy=DEFAULT_REFERRER_POLICY,
            frame_options='SAMEORIGIN',
            frame_options_allow_from=None,
            session_cookie_secure=True,
            session_cookie_http_only=True)
    app.name = "response_operations_ui"

    CSRFProtect(app)

    # Load css and js assets
    assets = Environment(app)

    if app.config['DEBUG'] or app.config['TESTING']:
        assets.cache = False
        assets.manifest = None

    if not app.config['DEBUG']:
        app.wsgi_app = GCPLoadBalancer(app.wsgi_app)

    assets.url = app.static_url_path

    app.jinja_env.add_extension('jinja2.ext.do')

    app.register_blueprint(filter_blueprint)

    app.url_map.strict_slashes = False
    app.secret_key = app.config['RESPONSE_OPERATIONS_UI_SECRET']

    logger_initial_config(service_name='response-operations-ui', log_level=app.config['LOGGING_LEVEL'])
    logger = wrap_logger(logging.getLogger(__name__))
    logger.info('Logger created', log_level=app.config['LOGGING_LEVEL'])

    login_manager = LoginManager(app)
    login_manager.init_app(app)
    login_manager.login_view = "sign_in_bp.sign_in"

    @app.context_processor
    def inject_availability_message():

        redis_avail_msg = app.config['SESSION_REDIS']

        if len(redis_avail_msg.keys('AVAILABILITY_MESSAGE_RES_OPS')) == 1:
            return {
                "availability_message": redis_avail_msg.get('AVAILABILITY_MESSAGE_RES_OPS').decode('utf-8')
            }
        return {}

    @login_manager.user_loader
    def user_loader(user_id):
        username = session.get('username')
        return User(user_id, username)

    # wrap in the flask server side session manager and back it by redis
    app.config['SESSION_REDIS'] = redis.StrictRedis(host=app.config['REDIS_HOST'],
                                                    port=app.config['REDIS_PORT'],
                                                    db=app.config['REDIS_DB'])

    app.jinja_environment.lstrip_blocks = True

    if app.config['DEBUG'] or os.environ.get('JINJA_RELOAD'):
        app.jinja_env.auto_reload = True

    Session(app)

    setup_blueprints(app)

    return app
Beispiel #42
0
from flask_security import RoleMixin
from flask_security import SQLAlchemySessionUserDatastore
from flask_security.forms import PasswordField
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer
from sqlalchemy import String, TIMESTAMP
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
from sqlalchemy.orm import relationship, backref
from sqlalchemy.sql.expression import func
from structlog import wrap_logger
from wtforms import HiddenField

from console import db, app, settings
from console.helpers.exceptions import UserExistsError, UserCreationError

logger = wrap_logger(logging.getLogger(__name__))


class RolesUsers(db.Model):
    __tablename__ = 'roles_users'
    id = Column(Integer(), primary_key=True)
    user_id = Column('user_id', Integer(), ForeignKey('flaskuser.id'))
    role_id = Column('role_id', Integer(), ForeignKey('role.id'))


class Role(db.Model, RoleMixin):
    __tablename__ = 'role'
    id = Column(Integer(), primary_key=True)
    name = Column(String(80), unique=True)
    description = Column(String(255))
import logging

from structlog import wrap_logger
from structlog import PrintLogger

logger = wrap_logger(logging.getLogger("iac-stub"))


class Iac_Stub:
    def __init__(self):
        self.iac_fixed_response = None

    def get_iac_stub(self, IacEntered):
        """Create the fixed iac response as a Python dictionary object. This can then be converted to a json object later on."""
        PrintLogger().info("Now in the get_iac_stub function")
        iac_fixed_response = {
            'caseId': '0337c579-ce9d-4357-a620-5e4c565cfac1',
            'caseRef': '1000000000000002',
            'iac': IacEntered,
            'active': True,
            'questionSet': None,
            'lastUsedDateTime': 1543502892050
        }
        PrintLogger().info("Now returning the iac fixed response: " +
                           repr(iac_fixed_response))
        return iac_fixed_response