def run(): logging.basicConfig(format=settings.LOGGING_FORMAT, datefmt="%Y-%m-%dT%H:%M:%S", level=settings.LOGGING_LEVEL) logging.getLogger('sdc.rabbit').setLevel(logging.INFO) # These structlog settings allow bound fields to persist between classes structlog.configure(logger_factory=LoggerFactory(), context_class=wrap_dict(dict)) logger = structlog.getLogger() logger.info('Starting SDX Downstream', version=__version__) message_processor = MessageProcessor() quarantine_publisher = QueuePublisher( urls=settings.RABBIT_URLS, queue=settings.RABBIT_QUARANTINE_QUEUE ) message_consumer = MessageConsumer( durable_queue=True, exchange=settings.RABBIT_EXCHANGE, exchange_type='topic', rabbit_queue=settings.RABBIT_QUEUE, rabbit_urls=settings.RABBIT_URLS, quarantine_publisher=quarantine_publisher, process=message_processor.process ) try: message_consumer.run() except KeyboardInterrupt: message_consumer.stop()
def __init__(self, id, server, address, description=None, path=None, protocol_bindings=None, available=True, authentication_required=False): self.id = id self.server = server self.address = address self.path = path self.description = description self.supported_protocol_bindings = ( protocol_bindings or self.supported_protocol_bindings) self.available = available self.authentication_required = authentication_required self.log = structlog.getLogger( "{}.{}".format(self.__module__, self.__class__.__name__), service_id=id) if not self.supported_protocol_bindings: self.log.warning( "No protocol bindings specified, service will be invisible", service=self.id)
# Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname( os.path.abspath(__file__)))) import buildscripts.resmokelib.parser as _parser # pylint: disable=wrong-import-position import buildscripts.resmokelib.suitesconfig as suitesconfig # pylint: disable=wrong-import-position import buildscripts.util.read_config as read_config # pylint: disable=wrong-import-position import buildscripts.util.taskname as taskname # pylint: disable=wrong-import-position import buildscripts.util.teststats as teststats # pylint: disable=wrong-import-position # pylint: disable=wrong-import-position from buildscripts.patch_builds.task_generation import TimeoutInfo, resmoke_commands # pylint: enable=wrong-import-position LOGGER = structlog.getLogger(__name__) AVG_SETUP_TIME = int(timedelta(minutes=5).total_seconds()) DEFAULT_TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig", "suites") CONFIG_FILE = "./.evergreen.yml" MIN_TIMEOUT_SECONDS = int(timedelta(minutes=5).total_seconds()) MAX_EXPECTED_TIMEOUT = int(timedelta(hours=48).total_seconds()) LOOKBACK_DURATION_DAYS = 14 GEN_SUFFIX = "_gen" HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost. # This file was generated by {file} from # {suite_file}. """
import structlog from ...models import ( Item, AddItemRequest, AddItemResponse, GetItemResponse, ListItemsResponse, ) from fastapi import APIRouter, HTTPException from redis.sentinel import Sentinel router = APIRouter() logger = structlog.getLogger("redis-items") sentinel = Sentinel([('redis-0.redis', 26379)], socket_timeout=0.1) @router.put( "/{key}", summary="Create redis item", response_model=AddItemResponse, ) def add(key: str, req: AddItemRequest): logger.info("Adding redis item", key=key, value=req.value) master = sentinel.master_for( "redis") # slaves are read-only; use master for writes master.set(key, req.value) return AddItemResponse(key=key, value=req.value)
import atexit import datetime import os import shutil import tempfile import uuid from pathlib import Path from typing import List, Iterable, Union, Tuple import pathlib import structlog import logging from datacube.utils import is_supported_document_type, read_documents, InvalidDocException, uri_to_local_path _LOG = structlog.getLogger() # This check is buggy when used with Tuple[] type: https://github.com/PyCQA/pylint/issues/867 # pylint: disable=invalid-sequence-index # This may eventually go to a config file... # ".trash" directories will be created at this level for any datasets contained within. # TODO: Could these be inferred from the collections paths? BASE_DIRECTORIES = [ '/g/data/fk4/datacube', '/g/data/rs0/datacube', '/g/data/v10/reprocess', '/g/data/v10/archived', '/g/data/rs0/scenes', '/scratch/v10/scenes', '/g/data/v10/public/data',
import numpy as np import requests import structlog from beard.metrics import b3_f_score, b3_precision_recall_fscore from inspire_disambiguation import conf from inspire_disambiguation.core.es.readers import ( get_input_clusters, get_curated_signature_blocks, get_signatures) from inspire_disambiguation.core.helpers import (process_clustering_output, train_validation_split) from inspire_disambiguation.core.ml.models import (Clusterer, DistanceEstimator, EthnicityEstimator) from inspire_disambiguation.core.ml.sampling import sample_signature_pairs from redis import StrictRedis LOGGER = structlog.getLogger() def train_and_save_ethnicity_model(load_data_path, save_model_path): """Train the ethnicity estimator model and save it to disk. Args: load_data_path (str): Full path to training data for ethnicity estimator. save_model_path (str): Full path where trained ethnicity model will be saved. """ start_time = datetime.now() estimator = EthnicityEstimator() estimator.load_data(load_data_path) load_time = datetime.now() LOGGER.info("Training EthnicityEstimator. May take a while...") estimator.fit()
import typing as t import structlog as logging from contextlib import contextmanager from django.db import models from apps.b3_migration.model_descriptors.utils import ( get_model_class, ) from apps.b3_migration.sync.auto_synchronization_base \ import AutoSynchronizationBase logger = logging.getLogger(__name__) def sync_source_and_target_models( source_instance, source_model_descriptor, target_model_descriptor, buddy_model_class, logging_prefix, update=False, ): """ Create/update an instance of the target model based on the source model 1. Using the target model's :dict: `fields_mapping` ( which maps the names of fields in the source model to the name of fields in the new model e.g. fields_mapping = {'postcode': 'zip_code'} leads to
import click import structlog from click_didyoumean import DYMGroup from presence import config from presence.cli import cli from presence.rpc import Broker, Client, Worker, ServiceClient from presence.tasks.store import Store log = structlog.getLogger() @cli.group(cls=DYMGroup) def run(): pass @run.command() def broker(): """Run the broker""" port = config['port'] bind = 'tcp://*:{}'.format(port) log.info('Broker starting...', bind=bind) broker = Broker(bind) broker.start() @run.command()
"""Package initializer.""" from pyramid.config import Configurator from pyramid.router import Router from pyramid_heroku import expandvars_dict import structlog import typing as t logger = structlog.getLogger("init") def configure_logging() -> None: """Configure structlog logging. Whenever structlog is imported it only creates a proxy. Calling this method turns the proxy into an actual logger. """ structlog.configure( processors=[ structlog.stdlib.filter_by_level, structlog.stdlib.add_log_level, structlog.stdlib.add_logger_name, structlog.processors.format_exc_info, structlog.processors.KeyValueRenderer(key_order=("level", "logger", "event"), sort_keys=True), ], logger_factory=structlog.stdlib.LoggerFactory(), cache_logger_on_first_use=True, )
# -*- coding: utf-8 -*- """ Sendgrid transport class """ import structlog import toolz from sendgrid import SendGridAPIClient from sendgrid.helpers.mail import Content from sendgrid.helpers.mail import Email from sendgrid.helpers.mail import Mail from services.notification_sender.email_templates import EmailRenderer from .base_transport import BaseTransport logger = structlog.getLogger(__name__, transport='SendGridTranport', transport_type='email') async def send_email(notification): asyncio.sleep(1) async def handle_email_transport(pool): conn = await pool.acquire() while True: loop_start = time.perf_counter() # begin transaction transport_type = TransportType['email'] loop_start = time.perf_counter() # begin transaction async with QItem(conn, transport_type=transport_type) as item:
from typing import List from sqlalchemy.exc import IntegrityError, DataError, ProgrammingError from sqlalchemy.orm import sessionmaker, Session from structlog import getLogger from db.models.base import BaseModel log = getLogger('DataBase') class DBIntegrityException(Exception): pass class DBDuplicateException(Exception): pass class DBDataException(Exception): pass class DBNoResultException(Exception): pass class DBProgrammingException(Exception): pass
import sys import uuid import structlog import twisted from twisted.internet import protocol, reactor logger = structlog.getLogger() class Counter(object): i = 0 def inc(self): self.i += 1 def __repr__(self): return str(self.i) class Echo(protocol.Protocol): def connectionMade(self): self._counter = Counter() self._log = logger.new( connection_id=str(uuid.uuid4()), peer=self.transport.getPeer().host, count=self._counter, ) def dataReceived(self, data):
stdlib.PositionalArgumentsFormatter(), processors.TimeStamper(fmt="iso"), processors.StackInfoRenderer(), processors.format_exc_info, processors.UnicodeDecoder(), stdlib.render_to_log_kwargs ]) LOGLEVEL = None log = None with open('logger/config.json') as f: dictConfig = json.loads(f.read()) conf = env_config.CONFIG if conf["debug"] == True: LOGLEVEL = logging.DEBUG else: LOGLEVEL = logging.INFO if conf["log_file_handler"] == True: os.makedirs('/tmp/.event_server', exist_ok=True) dictConfig["handlers"]["json"]["class"] = 'logging.FileHandler' dictConfig["handlers"]["json"][ "filename"] = '/tmp/.event_server/server.log' else: dictConfig["handlers"]["json"]["class"] = 'logging.StreamHandler' dictConfig["loggers"][""]["level"] = LOGLEVEL logging.config.dictConfig(dictConfig) log = structlog.getLogger(conf["system_log_code"]) log.debug("logging initialized")
def map_fips(self, fips: str) -> None: """ For a given fips code, for either a county or state, generate the CAN UI output format. Parameters ---------- fips: str FIPS code to map. """ log.info("Mapping output to WebUI.", state=self.state, fips=fips) shim_log = structlog.getLogger(fips=fips) pyseir_outputs = load_data.load_ensemble_results(fips) try: fit_results = load_inference_result(fips) t0_simulation = datetime.fromisoformat(fit_results["t0_date"]) except (KeyError, ValueError): log.error("Fit result not found for fips. Skipping...", fips=fips) return population = self._get_population(fips) # We will shim all suppression policies by the same amount (since historical tracking error # for all policies is the same). baseline_policy = "suppression_policy__inferred" # This could be any valid policy # We need the index in the model's temporal frame. idx_offset = int(fit_results["t_today"] - fit_results["t0"]) # Get the latest observed values to use in calculating shims observed_latest_dict = combined_datasets.get_us_latest_for_fips(fips) observed_death_latest = observed_latest_dict[CommonFields.DEATHS] observed_total_hosps_latest = observed_latest_dict[ CommonFields.CURRENT_HOSPITALIZED] observed_icu_latest = observed_latest_dict[CommonFields.CURRENT_ICU] # For Deaths model_death_latest = pyseir_outputs[baseline_policy]["total_deaths"][ "ci_50"][idx_offset] model_acute_latest = pyseir_outputs[baseline_policy]["HGen"]["ci_50"][ idx_offset] model_icu_latest = pyseir_outputs[baseline_policy]["HICU"]["ci_50"][ idx_offset] model_total_hosps_latest = model_acute_latest + model_icu_latest death_shim = shim.calculate_strict_shim( model=model_death_latest, observed=observed_death_latest, log=shim_log.bind(type=CommonFields.DEATHS), ) total_hosp_shim = shim.calculate_strict_shim( model=model_total_hosps_latest, observed=observed_total_hosps_latest, log=shim_log.bind(type=CommonFields.CURRENT_HOSPITALIZED), ) # For ICU This one is a little more interesting since we often don't have ICU. In this case # we use information from the same aggregation level (intralevel) to keep the ratios # between general hospitalization and icu hospitalization icu_shim = shim.calculate_intralevel_icu_shim( model_acute=model_acute_latest, model_icu=model_icu_latest, observed_icu=observed_icu_latest, observed_total_hosps=observed_total_hosps_latest, log=shim_log.bind(type=CommonFields.CURRENT_ICU), ) # Iterate through each suppression policy. # Model output is interpolated to the dates desired for the API. suppression_policies = [ key for key in pyseir_outputs.keys() if key.startswith("suppression_policy") ] for suppression_policy in suppression_policies: output_for_policy = pyseir_outputs[suppression_policy] output_model = pd.DataFrame() t_list = output_for_policy["t_list"] t_list_downsampled = range(0, int(max(t_list)), self.output_interval_days) output_model[schema.DAY_NUM] = t_list_downsampled output_model[schema.DATE] = [ (t0_simulation + timedelta(days=t)).date().strftime("%Y-%m-%d") for t in t_list_downsampled ] output_model[schema.TOTAL] = population output_model[schema.TOTAL_SUSCEPTIBLE] = np.interp( t_list_downsampled, t_list, output_for_policy["S"]["ci_50"]) output_model[schema.EXPOSED] = np.interp( t_list_downsampled, t_list, output_for_policy["E"]["ci_50"]) output_model[schema.INFECTED] = np.interp( t_list_downsampled, t_list, np.add(output_for_policy["I"]["ci_50"], output_for_policy["A"]["ci_50"]), ) # Infected + Asympt. output_model[schema.INFECTED_A] = output_model[schema.INFECTED] interpolated_model_acute_values = np.interp( t_list_downsampled, t_list, output_for_policy["HGen"]["ci_50"]) output_model[schema.INFECTED_B] = interpolated_model_acute_values raw_model_icu_values = output_for_policy["HICU"]["ci_50"] interpolated_model_icu_values = np.interp(t_list_downsampled, t_list, raw_model_icu_values) output_model[schema.INFECTED_C] = ( icu_shim + interpolated_model_icu_values).clip(min=0) # General + ICU beds. don't include vent here because they are also counted in ICU output_model[schema.ALL_HOSPITALIZED] = ( interpolated_model_acute_values + interpolated_model_icu_values + total_hosp_shim).clip(min=0) output_model[schema.ALL_INFECTED] = output_model[schema.INFECTED] # Shim Deaths to Match Observed raw_model_deaths_values = output_for_policy["total_deaths"][ "ci_50"] interp_model_deaths_values = np.interp(t_list_downsampled, t_list, raw_model_deaths_values) output_model[schema.DEAD] = (interp_model_deaths_values + death_shim).clip(min=0) # Continue mapping final_beds = np.mean(output_for_policy["HGen"]["capacity"]) output_model[schema.BEDS] = final_beds output_model[schema.CUMULATIVE_INFECTED] = np.interp( t_list_downsampled, t_list, np.cumsum(output_for_policy["total_new_infections"]["ci_50"]), ) if fit_results: output_model[schema.Rt] = np.interp( t_list_downsampled, t_list, fit_results["eps2"] * fit_results["R0"] * np.ones(len(t_list)), ) output_model[schema.Rt_ci90] = np.interp( t_list_downsampled, t_list, 2 * fit_results["eps2_error"] * fit_results["R0"] * np.ones(len(t_list)), ) else: output_model[schema.Rt] = 0 output_model[schema.Rt_ci90] = 0 output_model[schema.CURRENT_VENTILATED] = ( icu_shim + np.interp(t_list_downsampled, t_list, output_for_policy["HVent"]["ci_50"])).clip(min=0) output_model[schema.POPULATION] = population # Average capacity. output_model[schema.ICU_BED_CAPACITY] = np.mean( output_for_policy["HICU"]["capacity"]) output_model[schema.VENTILATOR_CAPACITY] = np.mean( output_for_policy["HVent"]["capacity"]) # Truncate date range of output. output_dates = pd.to_datetime(output_model["date"]) output_model = output_model[ (output_dates >= datetime(month=3, day=3, year=2020)) & (output_dates < datetime.today() + timedelta(days=90))] output_model = output_model.fillna(0) # Fill in results for the Rt indicator. rt_results = load_Rt_result(fips) if rt_results is not None: rt_results.index = rt_results[ "Rt_MAP_composite"].index.strftime("%Y-%m-%d") merged = output_model.merge( rt_results[["Rt_MAP_composite", "Rt_ci95_composite"]], right_index=True, left_on="date", how="left", ) output_model[schema.RT_INDICATOR] = merged["Rt_MAP_composite"] # With 90% probability the value is between rt_indicator - ci90 # to rt_indicator + ci90 output_model[schema.RT_INDICATOR_CI90] = ( merged["Rt_ci95_composite"] - merged["Rt_MAP_composite"]) else: log.warning( "No Rt Results found, clearing Rt in output.", fips=fips, suppression_policy=suppression_policy, ) output_model[schema.RT_INDICATOR] = "NaN" output_model[schema.RT_INDICATOR_CI90] = "NaN" output_model[[schema.RT_INDICATOR, schema.RT_INDICATOR_CI90]] = output_model[[ schema.RT_INDICATOR, schema.RT_INDICATOR_CI90 ]].fillna("NaN") int_columns = [ col for col in output_model.columns if col not in ( schema.DATE, schema.Rt, schema.Rt_ci90, schema.RT_INDICATOR, schema.RT_INDICATOR_CI90, schema.FIPS, ) ] output_model.loc[:, int_columns] = output_model[int_columns].fillna( 0).astype(int) output_model.loc[:, [ schema.Rt, schema.Rt_ci90, schema.RT_INDICATOR, schema. RT_INDICATOR_CI90 ]] = output_model[[ schema.Rt, schema.Rt_ci90, schema.RT_INDICATOR, schema.RT_INDICATOR_CI90 ]].fillna(0) output_model[schema.FIPS] = fips intervention = Intervention.from_webui_data_adaptor( suppression_policy) output_model[schema.INTERVENTION] = intervention.value output_path = get_run_artifact_path(fips, RunArtifact.WEB_UI_RESULT, output_dir=self.output_dir) output_path = output_path.replace("__INTERVENTION_IDX__", str(intervention.value)) output_model.to_json(output_path, orient=OUTPUT_JSON_ORIENT)
# Installed import datetime import http import flask import json import jwcrypto from jwcrypto import jwk, jwt import structlog # Own modules from dds_web import basic_auth, auth, mail from dds_web.errors import AuthenticationError, AccessDeniedError, InviteError, TokenMissingError from dds_web.database import models import dds_web.utils action_logger = structlog.getLogger("actions") # VARIABLES ############################################################################ VARIABLES # MFA_EXPIRES_IN = datetime.timedelta(hours=168) #################################################################################################### # FUNCTIONS ############################################################################ FUNCTIONS # #################################################################################################### # Error handler -------------------------------------------------------------------- Error handler # @basic_auth.error_handler def auth_error(status): """Handles 401 (Unauthorized) or 403 (Forbidden) for basic authentication."""
def format(self, record): for arg in ('exception', '_name'): val = getattr(record, arg, None) if isinstance(val, bytes): setattr(record, arg, val.decode('utf-8')) return super(ConsoleFormatter, self).format(record) structlog.configure( processors=[ structlog.stdlib.filter_by_level, structlog.stdlib.PositionalArgumentsFormatter(), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, structlog.processors.UnicodeEncoder(), simulate_stdlib_logging ], context_class=dict, logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, cache_logger_on_first_use=True, ) LOGGER = structlog.getLogger('assembl') def includeme(config): """add request.logger""" config.add_request_method( 'assembl.lib.logging.logger_for_request', 'logger')
import structlog from .test_base import BaseTestCase from tasktiger import Worker from tasktiger.logging import tasktiger_processor from .utils import get_tiger tiger = get_tiger() logger = structlog.getLogger("tasktiger") def logging_task(): log = logger.info("simple task") # Confirm tasktiger_processor injected task id assert log[1]["task_id"] == tiger.current_task.id class TestLogging(BaseTestCase): """Test logging.""" def test_structlog_processor(self): try: # Use ReturnLogger for testing structlog.configure( processors=[tasktiger_processor], context_class=dict, logger_factory=structlog.ReturnLoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger,
def __init__(self, worker_port=None, log_directory=None): threading.Thread.__init__(self) structlog.configure( processors=[ structlog.stdlib.filter_by_level, structlog.stdlib.add_logger_name, structlog.stdlib.add_log_level, structlog.stdlib.PositionalArgumentsFormatter(), structlog.processors.TimeStamper(fmt='iso'), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, structlog.processors.JSONRenderer() # , # zmq_processor ], context_class=dict, logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, cache_logger_on_first_use=True, ) # create file handler which logs messages down to the debug level to a file for postprocessing if log_directory is None: latest_experiment = logging.FileHandler('../logs/octrl.log', mode='w') else: latest_experiment = logging.FileHandler(log_directory+'/octrl.log', mode='w') latest_experiment.setLevel(logging.DEBUG) # create console handler with a higher log level console = logging.StreamHandler(sys.stderr) console.setLevel(logging.WARNING) if log_directory is None: rotating = handlers.RotatingFileHandler('../logs/experiment_history.log', mode='w', maxBytes=128e+6, backupCount=5, delay=True) else: rotating = handlers.RotatingFileHandler(log_directory+'/experiment_history.log', mode='w', maxBytes=128e+6, backupCount=5, delay=True) self.logger = structlog.getLogger() self.logger.setLevel(logging.DEBUG) try: rotating.doRollover() except AttributeError: # python2.6 quirk pass rotating.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') console.setFormatter(formatter) self.logger.addHandler(latest_experiment) self.logger.addHandler(console) self.logger.addHandler(rotating) if worker_port is None: self.worker_port = 5555 + 128 else: self.worker_port = worker_port self.logger.info('Logging thread started...') self.logger.info('Logging initialized in {0} on port {1}'.format(threading.current_thread().name, self.worker_port)) self.context = None self.socket = None
"""Public API application module.""" import structlog from aiohttp import web from db import db_manager from api.middlewares import error_middleware, auth_middleware from api.v1.handlers import (ping, auth, register, check) logger = structlog.getLogger('api.' + __name__) def init_server(loop=None): """Configure server before starting. :param loop: Event loop :return: Server app """ app = web.Application(middlewares=[ error_middleware, web.normalize_path_middleware( append_slash=True, merge_slashes=True, redirect_class=web.HTTPPermanentRedirect), auth_middleware ]) add_routes(app) async def init(app): """Configure app before starting server.
""" from datetime import datetime from getoffmylawn.auth.models import User from getoffmylawn.urls.models import Url from pyramid.paster import bootstrap from pyramid.paster import setup_logging from sqlalchemy.orm.session import Session import argparse import structlog import sys import transaction import typing as t logger = structlog.getLogger("populate") USER_ONE_ID = "aaaaaaaa-bbbb-4ccc-aaaa-eeeeeeeeeee1" USER_TWO_ID = "aaaaaaaa-bbbb-4ccc-aaaa-eeeeeeeeeee2" USER_JOHNJACOB_ID = "aaaaaaaa-bbbb-4ccc-aaaa-eeeeeeeeeee3" URL_FOO_ID = "aaaaaaaa-bbbb-4ccc-aaaa-eeeeeeeeeee1" URL_BAR_ID = "aaaaaaaa-bbbb-4ccc-aaaa-eeeeeeeeeee2" URL_JOHNJACOB_ID = "aaaaaaaa-bbbb-4ccc-aaaa-eeeeeeeeeee3" # "secret", hashed SECRET = "$argon2i$v=19$m=512,t=2,p=2$mRMCwLg3Rgih1JqTUooxxg$/bBw6iXly9rfryTkaoPX/Q" def add_users(db: Session) -> None: """Add demo users to db."""
""" from __future__ import ( absolute_import, division, print_function, unicode_literals ) import logging.config import structlog from distutils.version import StrictVersion from config import logging_config from config.service_name import MICROSERVICE_NAME from local_mqtt_client.local_mqtt_client import LocalMQTTClient version = StrictVersion("1.0.1") # load the config config logging.config.dictConfig(logging_config.get_logging_conf()) mosquito_monitor_logger = structlog.getLogger(MICROSERVICE_NAME) mosquito_monitor_logger.addHandler(logging.NullHandler()) mosquito_monitor_logger.info("Starting the Monitor") lbc = LocalMQTTClient() mosquito_monitor_logger.info("UpstreamMQTTClient Object Created") lbc.run_loop(in_thread=False, forever=True)
from typing import TypeVar from asyncpg.pool import Pool from asyncpg.connection import Connection from asyncio.queues import Queue from time import perf_counter from sqlalchemy.dialects.postgresql import JSONB import asyncpg import yo.db from yo.db import metadata from yo.schema import TransportType from yo.json import loads logger = structlog.getLogger(__name__, source='YoDB') PoolOrConn = TypeVar('PoolOrConn', Pool, Connection) QItemId = int NotificationId = int class QueueStorage: '''A mixin class to preserve compatability with asyncio.Queue which calls len(self._queue) ''' def __init__(self, loop, pool): self.loop = loop self.pool = pool def __len__(self) -> int:
from __future__ import absolute_import import structlog from tasktiger import TaskTiger, Worker from tasktiger.logging import tasktiger_processor from .test_base import BaseTestCase from .utils import get_tiger, get_redis tiger = get_tiger() logger = structlog.getLogger("tasktiger") def logging_task(): log = logger.info("simple task") # Confirm tasktiger_processor injected task id assert log[1]["task_id"] == tiger.current_task.id class TestLogging(BaseTestCase): """Test logging.""" def test_structlog_processor(self): try: # Use ReturnLogger for testing structlog.configure( processors=[tasktiger_processor], context_class=dict, logger_factory=structlog.ReturnLoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger,
# SPDX-FileCopyrightText: 2017 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 import socket import structlog from decisionengine.framework.modules.logging_configDict import DELOGGER_CHANNEL_NAME, LOGGERNAME logger = structlog.getLogger(LOGGERNAME) logger = logger.bind(module=__name__.split(".")[-1], channel=DELOGGER_CHANNEL_NAME) def get_random_port(): try: logger.debug("looking for random port in get_random_port") with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("127.0.0.1", 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) return s.getsockname()[1] except OSError: # pragma: no cover logger.exception("problem with get_random_port") raise except Exception: # pragma: no cover logger.exception("Unexpected error!") raise
from structlog import getLogger logger = getLogger(__name__)
@dataclass class Color: """A color container object. Default color is black. """ r: float = 0 #: Normalized red channel value. g: float = 0 #: Normalized green channel value. b: float = 0 #: Normalized blue channel value. a: float = 1 #: Normalized alpha value (influences the transparency). logger = structlog.getLogger(__name__) class Visualizer: """A class that creates different types of visualizations. """ def draw_gantt(self, stats: SchedulerStats, filepath: str): """Draws a Gantt chart. Directories referenced by filepath are created similar to mkdir -p. Args: stats (SchedulerStats): A container object for the scheduler's \ output statistics. filepath (str): The location for writing the resulting Gantt chart.
import structlog import requests from functools import lru_cache from telegram import InputMediaPhoto from eduzen_bot.plugins.commands.series import keyboards from eduzen_bot.plugins.commands.series.constants import EZTV_NO_RESULTS, EZTV_API_ERROR from eduzen_bot.plugins.commands.series.api import prettify_serie, get_all_seasons, get_poster_url logger = structlog.getLogger(filename=__name__) def monospace(text): return f"```\n{text}\n```" def prettify_episode(ep): """Episodes have name, season, episode, torrent, magnet, size, seeds and released attributes""" # Some episodes do not have a torrent download. But they do have a magnet link. # Since magnet links are not clickable on telegram, we leave them as a fallback. if ep.torrent: header = f"[{ep.name}]({ep.torrent})\n" elif ep.magnet: header = f"Magnet: {monospace(ep.magnet)}" else: header = "No torrent nor magnet available for this episode." return f"{header}" f"🌱 Seeds: {ep.seeds} | 🗳 Size: {ep.size or '-'}"
import click import structlog import collections from datacube import Datacube from datacube.model import Dataset from datacube.ui import click as ui from digitalearthau import uiutil _LOG = structlog.getLogger('archive-locationless') _siblings_count = 0 @click.command() @click.option('--check-locationless/--no-check-locationless', is_flag=True, default=False, help='Check any datasets without locations') @click.option('--archive-locationless', is_flag=True, default=False, help="Archive datasets with no active locations (forces --check-locationless)") @click.option('--check-ancestors', is_flag=True, default=False, help='Check if ancestor/source datasets are still active/valid') @click.option('--check-siblings', is_flag=True, default=False, help='Check if ancestor datasets have other children of the same type (they may be duplicates)') @click.option('--archive-siblings',
from __future__ import print_function from builtins import * # python3 types __all__ = ['file_get_bytes', 'file_put_bytes', 'file_truncate', 'file_update_attrs', 'simple_get_record',] import os import sys from time import sleep from time import time import structlog logger = structlog.getLogger('fuse.log-mixin.' + __name__) mylog = logger.bind(scope=__name__) import inspect # If we are running from the source directory, try # to load the module from there first. basedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # zzz print('{} init: argv:{}, basedir:{}'.format(os.path.basename(basedir), # sys.argv[0], # basedir,)) if (os.path.exists(basedir) and os.path.exists(os.path.join(basedir, 'setup.py'))): add_dirs = [os.path.join(basedir, os.path.basename(basedir)), os.path.join(basedir, '../si446x'), os.path.join(basedir, '../tagnet')] for ndir in add_dirs:
def setUp(self): self.factory = RequestFactory() self.logger = structlog.getLogger(__name__)
from funcy import flatten from ...db.notifications import create_notification from ...db import create_asyncpg_pool from .handlers import handle_vote from .handlers import handle_account_update from .handlers import handle_send from .handlers import handle_receive from .handlers import handle_follow from .handlers import handle_repost from .handlers import handle_power_down from .handlers import handle_mention from .handlers import handle_comment logger = structlog.getLogger(__name__, service_name='blockchain_follower') asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) logger = logger.bind() EXECUTOR = ThreadPoolExecutor() ''' { "block": 20000000, "op": [ "author_reward", { "author": "ivelina89", "permlink": "friends-forever", "bbd_payout": "2.865 BBD", "dpay_payout": "0.000 BEX", "vesting_payout": "1365.457442 VESTS" }
def successful_task(foo=None): import structlog logger = structlog.getLogger(__name__) logger.info("This is a successful task")
from sanic.exceptions import SanicException from sanic.request import Request from structlog import getLogger from api.base import APIValidateException from db.database import DataBase, DBSession, DBNoResultException from transport.sanic.base import SanicEndpoint log = getLogger('BaseSanicEndpoint') class BaseSanicEndpoint(SanicEndpoint): async def _method(self, request: Request, body: dict, *args, **kwargs): try: database: DataBase = self.context.database session: DBSession = database.make_session() return await super()._method(request, body, session=session) except APIValidateException as e: return await self.make_response_json(code=e.status_code, message=str(e)) except DBNoResultException as e: log.error(e) return await self.make_response_json(code=400) except SanicException as e: log.error(e) if hasattr(e, 'error_code'): return await self.make_response_json(code=e.status_code, error_code=e.error_code, message=str(e))
import json from jsonschema import SchemaError, ValidationError, validate from mock import patch from sdc.crypto.decrypter import decrypt from structlog import getLogger from tests.integration.integration_test_case import IntegrationTestCase from app.keys import KEY_PURPOSE_SUBMISSION logger = getLogger() FEEDBACK_FORM_URL = '/feedback' FEEDBACK_THANKYOU_URL = '/feedback/thank-you' class Feedback(IntegrationTestCase): def setUp(self): self.patcher = patch('app.setup.LogSubmitter') mock_class = self.patcher.start() self.instance = mock_class.return_value self.instance.send_message.return_value = True super().setUp() self.launchSurvey('test', 'textfield') def tearDown(self): self.patcher.stop() def test_correct_feeback_link_in_page(self): soup = self.getHtmlSoup()
class Help(HelpView): """ Route: help This takes care of the help section, sending the data off to the relevant functions. This can only be reached if you are logged in. """ # Specify the method(s) that are allowed to be used to reach this view methods = ["POST"] # Logger instance logger = logging.getLogger("netsocadmin.help") def dispatch_request(self) -> str: email = flask.request.form.get("email", "") subject = flask.request.form.get("subject", "") message = flask.request.form.get("message", "") # Ensure all fields are populated if not all([email, subject, message]): self.logger.info("not all fields specified") return self.render(help_error="Please specify all fields", help_active=True) sent_email, sent_discord = True, True # Send the email try: email_resp = help_post.send_help_email(flask.session["username"], email, subject, message) if not str(email_resp.status_code).startswith("20"): self.logger.error( f"non 20x status code for help email: {email_resp.status_code} - {email_resp.body}" ) sent_email = str(email_resp.status_code).startswith("20") except Exception as e: self.logger.error(f"failed to send help email: {str(e.body)}") # Try and send to Discord try: sent_discord = help_post.send_help_webhook( flask.session["username"], email, subject, message) except Exception as e: self.logger.error(f"failed to send help discord webhook: {str(e)}") # Check that at least one form of communication was sent if not sent_email and not sent_discord: # If not, report an error to the user return self.render( help_error= "There was a problem :( Please email [email protected] instead", help_active=True, ) # Otherwise when things are okay, report back stating so message = '' if sent_email: message += "sent help email" if sent_discord and sent_email: message += " and " if sent_discord: message += "fired discord webhook" self.logger.info(message) return self.render(help_success=True, help_active=True)
import anyconfig import argparse import structlog from opentaxii.taxii.entities import CollectionEntity from opentaxii.cli import app log = structlog.getLogger(__name__) def create_services(): parser = argparse.ArgumentParser( description="Create services using OpenTAXII Persistence API", formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( "-c", "--services-config", dest="config", help="YAML file with services configuration", required=True) args = parser.parse_args() services_config = anyconfig.load(args.config, forced_type="yaml") with app.app_context(): app.taxii_server.persistence.create_services_from_object( services_config) def create_collections():
], fmt=fmt, datefmt=datefmt ) def format(self, record): return super(ConsoleFormatter, self).format(record) structlog.configure( processors=[ structlog.stdlib.filter_by_level, structlog.stdlib.PositionalArgumentsFormatter(), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, structlog.processors.UnicodeEncoder(), simulate_stdlib_logging ], context_class=dict, logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, cache_logger_on_first_use=True, ) LOGGER = structlog.getLogger('assembl') def includeme(config): """add request.logger""" config.add_request_method( 'assembl.lib.logging.logger_for_request', 'logger')