def __init__(self, database_config): connection_string = self.__create_connection_string(database_config) engine = create_engine(connection_string) Base.metadata.create_all(engine) session = sessionmaker(bind=engine) self.session = session() self.logger = structlog.get_logger()
def __init__(self, notifier_config): self.logger = structlog.get_logger() self.twilio_configured = self.__validate_required_config('twilio', notifier_config) if self.twilio_configured: self.twilio_client = TwilioNotifier( twilio_key=notifier_config['twilio']['required']['key'], twilio_secret=notifier_config['twilio']['required']['secret'], twilio_sender_number=notifier_config['twilio']['required']['sender_number'], twilio_receiver_number=notifier_config['twilio']['required']['receiver_number'] ) self.slack_configured = self.__validate_required_config('slack', notifier_config) if self.slack_configured: self.slack_client = SlackNotifier( slack_key=notifier_config['slack']['required']['key'], slack_channel=notifier_config['slack']['required']['channel'] ) self.gmail_configured = self.__validate_required_config('gmail', notifier_config) if self.gmail_configured: self.gmail_client = GmailNotifier( username=notifier_config['gmail']['required']['username'], password=notifier_config['gmail']['required']['password'], destination_addresses=notifier_config['gmail']['required']['destination_emails'] ) self.integram_configured = self.__validate_required_config('integram', notifier_config) if self.integram_configured: self.integram_client = IntegramNotifier( url=notifier_config['integram']['required']['url'] )
def run(host, port, shell_host, shell_port, project_config, log, application_factory=web.Application, config_factory=Config, config_source_factory=YamlFileConfigSource, push_handler_factory=PushHandler, repo_manager_factory=RepoManager, cmd_factory=MergeitShell, telnet_shell_factory=CmdTelsh, telnet_server_factory=TelnetServer): logger = get_logger() loop = asyncio.get_event_loop() app = application_factory() config = config_factory(config_source_factory(project_config)) init_logging(log, config.get('name')) app.router.add_route('POST', '/push', lambda request: gitlab_push(request, config)) logger.info('application_start') loop.run_until_complete(loop.create_server(app.make_handler(), host, port)) shell = cmd_factory(config=config, push_handler_factory=push_handler_factory, repo_manager_factory=repo_manager_factory, forward=True) shell_factory = (lambda server, stream=TelnetShellStream, log=logging: telnet_shell_factory(server, stream, log, cmd=shell)) loop.run_until_complete( loop.create_server(lambda: telnet_server_factory(log=logging.getLogger(telnetlib3.__name__), shell=shell_factory), shell_host, shell_port) ) try: loop.run_forever() except KeyboardInterrupt: logger.info('application_interrupt')
def test_mozdef(app): sent = [] orig_MozDefEvent = mozdef_client.MozDefEvent with mock.patch('mozdef_client.MozDefEvent') as MozDefEvent: def constructor(target): msg = orig_MozDefEvent(target) msg.send = lambda: sent.append(msg) return msg MozDefEvent.side_effect = constructor logger = structlog.get_logger(__name__) logger.warn("unseen") logger.warn("test message", mozdef=True) logger.warn("with attr", attr="foo", mozdef=True) # check that 'unseen' wasn't seen, since mozdef was not true eq_({m.summary for m in sent}, {"test message", "with attr"}) # check a few other fields in one of the messages msg = sent[0] eq_(msg.source, __name__) eq_(msg._severity, orig_MozDefEvent.SEVERITY_WARNING) eq_(msg.tags, ['relengapi']) # and verify that the attribute showed up in details assert any([m.details.get('attr') == 'foo' for m in sent])
def __init__(self, username, password, destination_addresses): self.logger = structlog.get_logger() smtp_server = 'smtp.gmail.com:587' self.smtp_handler = smtplib.SMTP(smtp_server) self.username = username self.password = password self.destination_addresses = ','.join(destination_addresses)
def attach_console(request): event_log = request.app.get('smartmob.event_log') or structlog.get_logger() # Must connect here with a WebSocket. if request.headers.get('Upgrade', '').lower() != 'websocket': pass # Resolve the process. processes = request.app.setdefault('smartmob.processes', {}) slug = request.match_info['slug'] if slug not in processes: raise web.HTTPNotFound # WebSocket handshake. stream = web.WebSocketResponse() yield from stream.prepare(request) # TODO: retrieve data from the process and pipe it to the WebSocket. # Strawboss implementation doesn't provide anything for this at the # moment, so we'll have to do this later. # Log the request. event_log.info('process.attach', slug=slug) # Close the WebSocket. yield from stream.close() # Required by the framework, but I don't know why. return stream
def main(): """Initializes the application """ # Load settings and create the config object config = Configuration() settings = config.settings # Set up logger logs.configure_logging(settings['log_level'], settings['log_mode']) logger = structlog.get_logger() # Configure and run configured behaviour. exchange_interface = ExchangeInterface(config.exchanges) notifier = Notifier(config.notifiers) behaviour = Behaviour( config, exchange_interface, notifier ) while True: behaviour.run(settings['market_pairs'], settings['output_mode']) logger.info("Sleeping for %s seconds", settings['update_interval']) time.sleep(settings['update_interval'])
def backup(): logger = get_logger(__name__).bind( action='backup_db' ) logger.info('starting') command = 'pg_dump -U ' \ '{username} -h {host} -p {port} -F tar -O -d {db_name} -f {file} -w' if settings.POSTGRES_PASSWORD: command = 'PGPASSWORD={password} ' + command file_path = '/usr/src/app/core/dump_file_{unique_id}.tar'.format(unique_id=uuid.uuid4()) full_command = command.format(password=settings.POSTGRES_PASSWORD, username=settings.POSTGRES_USER, host=settings.POSTGRES_HOST, port=settings.POSTGRES_PORT, file=file_path, db_name=settings.POSTGRES_DB ) logger.debug('full_command', command=full_command) os.system(full_command) logger.debug('finished_successfully') return file_path
def restore_db(file_name): logger = get_logger(__name__).bind( action='restore_db' ) logger.info('starting') command = "pg_restore --verbose --clean --no-acl --no-owner" \ " -h {host} -p {port} -U {username} -d {db_name} {file_path}" if settings.POSTGRES_PASSWORD: command = 'PGPASSWORD={password} ' + command file_path = "/usr/src/app/core/{}".format(file_name) full_command = command.format( password=settings.POSTGRES_PASSWORD, username=settings.POSTGRES_USER, host=settings.POSTGRES_HOST, port=settings.POSTGRES_PORT, file_path=file_path, db_name=settings.POSTGRES_DB ) logger.debug('full_command', command=full_command) os.system(full_command) logger.debug('finished successfully') #'PGPASSWORD=mysecretpassword pg_dump -U postgres -h db -p 5432 -d slash_air -w' # pg_dump -h db -p 5432 -F c -O -U postgres postgres > backup.dump # pg_restore --verbose --clean --no-acl --no-owner -h db -p 5432 -U postgres -d postgres backup.dump # /usr/src/app/core
def main(args): init_logger(level=args.log_level.upper()) GITHUB_ACCESS_TOKEN = os.environ.get('GITHUB_ACCESS_TOKEN', None) if GITHUB_ACCESS_TOKEN is None: print("Missing GITHUB_ACCESS_TOKEN!", file=sys.stderr) sys.exit(1) RTM_API_KEY = os.environ.get('RTM_API_KEY', None) RTM_SHARED_SECRET = os.environ.get('RTM_SHARED_SECRET', None) RTM_TOKEN = os.environ.get('RTM_TOKEN', None) if None in (RTM_API_KEY, RTM_SHARED_SECRET, RTM_TOKEN): print("Missing RTM_API_KEY, RTM_SHARED_SECRET or RTM_TOKEN!", file=sys.stderr) sys.exit(1) log = get_logger() log = log.bind(component="main") log.info("gh.init") gh = github.GitHub(access_token=GITHUB_ACCESS_TOKEN) log.info("gh.get_issues") issues = get_github_issues(gh, GITHUB_ISSUE_QUERIES) log.info("rtm.init") rtm = createRTM(RTM_API_KEY, RTM_SHARED_SECRET, RTM_TOKEN) log.info("complete_missing_issues.start") complete_missing_issues(rtm, issues) log.info("complete_missing_issues.finish") log.info("add_new_issues.start") add_new_issues(rtm, issues) log.info("add_new_issues.finish")
def get_logger(level=logging.DEBUG, name=None, stream=DEFAULT_STREAM): """Configure and return a logger with structlog and stdlib.""" wrap_dict_class = structlog.threadlocal.wrap_dict(dict) structlog.configure( processors=[ structlog.stdlib.filter_by_level, structlog.stdlib.add_log_level, structlog.stdlib.add_logger_name, structlog.stdlib.PositionalArgumentsFormatter(), structlog.processors.TimeStamper(fmt='iso'), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, structlog.processors.JSONRenderer(sort_keys=True) ], context_class=wrap_dict_class, logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, cache_logger_on_first_use=True) log = structlog.get_logger(name) if not _has_streamhandler(logging.getLogger(name), level=level, stream=stream): streamhandler = logging.StreamHandler(stream) streamhandler.setLevel(level) streamhandler.setFormatter(logging.Formatter(fmt=LOG_FORMAT)) log.addHandler(streamhandler) log.setLevel(level) return log
def __init__(self, core, logical_device): try: self.core = core self.local_handler = core.get_local_handler() self.logical_device_id = logical_device.id self.root_proxy = core.get_proxy('/') self.flows_proxy = core.get_proxy( '/logical_devices/{}/flows'.format(logical_device.id)) self.groups_proxy = core.get_proxy( '/logical_devices/{}/flow_groups'.format(logical_device.id)) self.self_proxy = core.get_proxy( '/logical_devices/{}'.format(logical_device.id)) self.flows_proxy.register_callback( CallbackType.POST_UPDATE, self._flow_table_updated) self.groups_proxy.register_callback( CallbackType.POST_UPDATE, self._group_table_updated) self.self_proxy.register_callback( CallbackType.POST_ADD, self._port_added) self.self_proxy.register_callback( CallbackType.POST_REMOVE, self._port_removed) self.port_proxy = {} self.event_bus = EventBusClient() self.packet_in_subscription = self.event_bus.subscribe( topic='packet-in:{}'.format(logical_device.id), callback=self.handle_packet_in_event) self.log = structlog.get_logger(logical_device_id=logical_device.id) self._routes = None except Exception, e: self.log.exception('init-error', e=e)
def get_github_issues(gh, queries): """ Return a dict of issues found matching specified filters. :param gh: github.GitHub client :param queries: A list containing individual search queries to perform :return: A dictionary whose elements are in the form of {'repo#000: Summary': {..data from github}} """ log = get_logger() log = log.bind(component='get_github_issues') issues = {} results = [] for query in queries: log.debug("gh.issues.get", q=query) items = gh.issues.get(**query) log.debug("gh.issues.result", q=query, result_count=len(items)) results += items for issue in results: title = "{repository[name]}#{number}: {title}".format(**issue).strip() log.debug("gh.parse_issue", title=title) if title not in issues: issues[title] = issue return issues
def send_invite_email(self): from sentry.utils.email import MessageBuilder context = { 'email': self.email, 'organization': self.organization, 'url': absolute_uri(reverse('sentry-accept-invite', kwargs={ 'member_id': self.id, 'token': self.token, })), } msg = MessageBuilder( subject='Join %s in using Sentry' % self.organization.name, template='sentry/emails/member-invite.txt', html_template='sentry/emails/member-invite.html', type='organization.invite', context=context, ) try: msg.send_async([self.get_email()]) except Exception as e: logger = get_logger(name='sentry.mail') logger.exception(e)
def delete_process(request): event_log = request.app.get('smartmob.event_log') or structlog.get_logger() # Resolve the process. processes = request.app.setdefault('smartmob.processes', {}) slug = request.match_info['slug'] try: process = processes[slug] except KeyError: raise web.HTTPNotFound # Log the request. event_log.info('process.delete', slug=slug) # Kill the process and wait for it to complete. process['stop'].set_result(None) try: yield from process['task'] except Exception: # TODO: be more accurate! pass # Erase bookkeeping. del processes[slug] # Format the response. return web.Response( content_type='application/json', body=json.dumps({ # ... }).encode('utf-8'), )
def __init__(self, **options): #: logging facility self.logger = get_logger() #: keep track of retry attempts self.retry_attempt = 0 #: retry limit self.retry_limit = options.get('retry_limit', 30) #: retry interval self.retry_interval = options.get('retry_interval', 3) #: rabbit mq connection self.connection = None #: rabbit mq channnels self.channels = dict() #: services callback self.services_callback = None #: keep track of whether or not we are retrying self.retry_mode = False
def emit(self, record, logger=get_logger()): # If anyone wants to use the 'extra' kwarg to provide context within # structlog, we have to strip all of the default attributes from # a record because the RootLogger will take the 'extra' dictionary # and just turn them into attributes. kwargs = { k: v for k, v in six.iteritems(vars(record)) if k not in throwaways and v is not None } kwargs.update({ 'level': record.levelno, 'event': record.msg, }) if record.args: # record.args inside of LogRecord.__init__ gets unrolled # if it's the shape `({},)`, a single item dictionary. # so we need to check for this, and re-wrap it because # down the line of structlog, it's expected to be this # original shape. if isinstance(record.args, (tuple, list)): kwargs['positional_args'] = record.args else: kwargs['positional_args'] = (record.args, ) logger.log(**kwargs)
def update_logging(instance_id, vcore_id): """ Add the vcore id to the structured logger :param vcore_id: The assigned vcore id :return: structure logger """ def add_exc_info_flag_for_exception(_, name, event_dict): if name == 'exception': event_dict['exc_info'] = True return event_dict def add_instance_id(_, __, event_dict): event_dict['instance_id'] = instance_id return event_dict def add_vcore_id(_, __, event_dict): event_dict['vcore_id'] = vcore_id return event_dict processors = [ add_exc_info_flag_for_exception, structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, add_instance_id, add_vcore_id, FluentRenderer(), ] structlog.configure(processors=processors) # Mark first line of log log = structlog.get_logger() log.info("updated-logger") return log
def __init__(self, handler, alloc_id, traffic_descriptor, entity_id, name=None, vont_ani=None, is_mock=False): super(OnuTCont, self).__init__(alloc_id, traffic_descriptor, name=name, vont_ani=vont_ani) self._handler = handler self._is_mock = is_mock self._entity_id = entity_id self.log = structlog.get_logger(device_id=handler.device_id, alloc_id=alloc_id)
def __init__(self, namespace=None, source=None): if not namespace: try: frame = inspect.currentframe(1) namespace = frame.f_globals['__name__'] except: namespace = 'unknown' self._log = structlog.get_logger(namespace=namespace)
def initialize(self, dispatcher, logger=None): '''Initialize the handler. :param dispatcher: a ~:class:`utils.dispatch.Dispatcher` instance :param logger: logger instance ''' self.dispatcher = dispatcher self.logger = logger or structlog.get_logger(__name__)
def obtain_logger(name): """Get's a logger and attaches the correct DCC compatible Handler. Args: name (str): Name of the logger to get / create. Returns: Logger: Logger. """ return structlog.get_logger(name)
def __init__(self, survey, ftpconn, base_url, endpoint_name): self.logger = get_logger() self.survey = survey self.tx_id = "" self._setup_logger() self.ftp = ftpconn self._base_url = base_url self._endpoint_name = endpoint_name
def _create_logger(request): logger = structlog.get_logger("warehouse.request") # This has to use **{} instead of just a kwarg because request.id is not # an allowed kwarg name. logger = logger.bind(**{"request.id": request.id}) return logger
def __init__(self, parent, **kwargs): # TODO: Weed out those properties supported by common 'Port' object assert parent, 'parent is None' assert 'port_no' in kwargs, 'Port number not found' self.log = structlog.get_logger(port_no=kwargs.get('port_no')) self.log.info('creating') self._port_no = kwargs.get('port_no') self._name = kwargs.get('name', 'nni-{}'.format(self._port_no)) self._port = None self._logical_port = None self._parent = parent self._sync_tick = 10.0 self._sync_deferred = None self._stats_tick = 5.0 self._stats_deferred = None self._deferred = None self._state = NniPort.State.INITIAL # Local cache of NNI configuration self._enabled = None self._ianatype = '<type xmlns:ianaift="urn:ietf:params:xml:ns:yang:iana-if-type">ianaift:ethernetCsmacd</type>' # And optional parameters # TODO: Currently cannot update admin/oper status, so create this enabled and active # self._admin_state = kwargs.pop('admin_state', AdminState.UNKNOWN) # self._oper_status = kwargs.pop('oper_status', OperStatus.UNKNOWN) self._admin_state = AdminState.ENABLED self._oper_status = OperStatus.ACTIVE self._label = kwargs.pop('label', 'NNI port {}'.format(self._port_no)) self._mac_address = kwargs.pop('mac_address', '00:00:00:00:00:00') # TODO: Get with JOT and find out how to pull out MAC Address via NETCONF # TODO: May need to refine capabilities into current, advertised, and peer self._ofp_capabilities = kwargs.pop('ofp_capabilities', OFPPF_100GB_FD | OFPPF_FIBER) self._ofp_state = kwargs.pop('ofp_state', OFPPS_LIVE) self._current_speed = kwargs.pop('current_speed', OFPPF_100GB_FD) self._max_speed = kwargs.pop('max_speed', OFPPF_100GB_FD) self._device_port_no = kwargs.pop('device_port_no', self._port_no) # Statistics self.rx_packets = 0 self.rx_bytes = 0 self.tx_packets = 0 self.tx_bytes = 0 self.rx_dropped = 0 self.rx_errors = 0 self.rx_bcast = 0 self.rx_mcast = 0 self.tx_dropped = 0 self.tx_bcast = 0 self.tx_mcast = 0
def __init__(self, stream, dispatcher, container): self._dispatcher = dispatcher self._stream = stream self._stream.read_until(FRAME_END, self._read_frame) self.container = container self.data = '' self.logger = structlog.get_logger(__name__)
def __init__(self, config, max_attempts=5, wait_time=0.5): super(IPMIToolProvider, self).__init__(config) # maximum number of times to try/validate a command self.max_attempts = max_attempts # wait time between attempts self.wait_time = wait_time self.log = structlog.get_logger()
def get(self, req): logger = get_logger(__name__) logger.info('request_data', data=req.GET) backup_list = self.list_backups() if req.GET.get('path'): self.restore_db(req.GET['path']) return render(req, 'home.html', {'backup_list': backup_list})
def __init__(self, behaviour_config, exchange_interface, strategy_analyzer, notifier, db_handler): self.logger = structlog.get_logger() self.behaviour_config = behaviour_config self.exchange_interface = exchange_interface self.strategy_analyzer = strategy_analyzer self.notifier = notifier self.db_handler = db_handler
def __call__(self, *args, **kwargs): if app.config["TESTING"]: # When testing, celery tasks are called eagerly, from the same thread # so don't push an app context, the request's app context is already there return self.run(*args, **kwargs) else: with app.app_context(): g.log = structlog.get_logger().new() return self.run(*args, **kwargs)
from django.shortcuts import get_object_or_404 from django.template.response import TemplateResponse from django.http import HttpResponse from django.core.paginator import Paginator, PageNotAnInteger, InvalidPage, EmptyPage from django.db.models import Q from ..views import staff_member_required from ...table.models import Table from ...decorators import user_trail import json from structlog import get_logger logger = get_logger(__name__) @staff_member_required def list(request): try: options = Table.objects.all().order_by('-id') page = request.GET.get('page', 1) paginator = Paginator(options, 10) try: options = paginator.page(page) except PageNotAnInteger: options = paginator.page(1) except InvalidPage: options = paginator.page(1) except EmptyPage: options = paginator.page(paginator.num_pages) data = {
"""p2 replication controller""" import copy from shutil import copyfileobj from time import time from structlog import get_logger from p2.components.replication.constants import (TAG_BLOB_SOURCE_UUID, TAG_REPLICATION_TARGET) from p2.core.components.base import ComponentController from p2.core.models import Blob, Volume LOGGER = get_logger() # pylint: disable=too-few-public-methods class ReplicationController(ComponentController): """Replicate Blobs 1:1 between volumes""" template_name = 'components/replication/card.html' form_class = 'p2.components.replication.forms.ReplicationForm' @property def target_volume(self): """Get Target volume""" return Volume.objects.get( pk=self.instance.tags.get(TAG_REPLICATION_TARGET)) def _get_target_blob(self, source_blob): target_volume = self.target_volume # Check if there's a blob that's our source UUID as attribute
def __init__(self): """Initializes Breakout class """ self.logger = structlog.get_logger()
from structlog import get_logger import os import json from urllib.parse import urlparse from onelogin.saml2.auth import OneLogin_Saml2_Auth from onelogin.saml2.settings import OneLogin_Saml2_Settings from onelogin.saml2.utils import OneLogin_Saml2_Utils from onelogin.saml2.idp_metadata_parser import OneLogin_Saml2_IdPMetadataParser from . import NEED_TO_SIGN_IN_MSG from .flash import flash saml_routes = RouteTableDef() logger = get_logger('fsdr-ui') # Check if the user is logged in async def is_logged_in(request): session = await get_session(request) return 'samlUserdata' in session async def get_role_id(request): session = await get_session(request) if not 'samlUserdata' in session: redirect_to_login(request) roleids = session['samlUserdata']['roleID'] # TODO extra checking here? return roleids[0]
import random import structlog log = structlog.get_logger() class Graph: def __init__(self, cost_matrix: list): self.matrix = cost_matrix self.rank = len(self.matrix) self.pheromone = [[1 / self.rank**2 for j in range(self.rank)] for i in range(self.rank)] class ACO: def __init__( self, ant_count: int, run_without_improvement: int, alpha: float, beta: float, rho: float, q: float, pheromone_strategy: str, ): """ :param ant_count: :param run_without_improvement: :param alpha: relative importance of pheromone (distance) :param beta: relative importance of heuristic information (pheromone)
from raiden.constants import NULL_ADDRESS from raiden.exceptions import ( AddressWrongContract, ContractVersionMismatch, TransactionThrew, UnknownAddress, ) from raiden.network.rpc.client import check_address_has_code from raiden.network.rpc.smartcontract_proxy import ContractProxy from raiden.network.rpc.transactions import check_transaction_threw from raiden.settings import EXPECTED_CONTRACTS_VERSION from raiden.utils import compare_versions, pex, privatekey_to_address from raiden_contracts.constants import CONTRACT_ENDPOINT_REGISTRY from raiden_contracts.contract_manager import ContractManager log = structlog.get_logger(__name__) # pylint: disable=invalid-name class Discovery: """On chain smart contract raiden node discovery: allows registering endpoints (host, port) for your ethereum-/raiden-address and looking up endpoints for other ethereum-/raiden-addressess. """ def __init__( self, jsonrpc_client, discovery_address, contract_manager: ContractManager, ): contract = jsonrpc_client.new_contract(
import structlog from aiohttp import web from urllib.parse import unquote from autofin import settings, models from autofin.contact import ContactMethod from autofin.error import capture_error, capture_error_context from autofin.billing import InvoiceRetriever from autofin.contact import MessageFormatter LOGGER = structlog.get_logger(__name__) async def on_sms_received(request): """HTTP handler for the webhook that is called when a SMS is received.""" try: context = dict(raw_body=await request.text()) capture_error_context(**context) logger = LOGGER.bind(**context) data = await request.post() from_ = data.get("From") if not from_: logger.error("Received bad request, missing from") return web.Response(status=400, text="Missing 'From'") body = data.get("Body")
from typing import Callable, Optional import numpy as np import structlog import torch from torch.optim import Optimizer from torch.utils.data import DataLoader, Dataset from torch.utils.data.dataloader import default_collate from tqdm import tqdm from baal.utils.array_utils import stack_in_memory from baal.utils.cuda_utils import to_cuda from baal.utils.iterutils import map_on_tensor from baal.utils.metrics import Loss log = structlog.get_logger("ModelWrapper") def _stack_preds(out): if isinstance(out[0], Sequence): out = [torch.stack(ts, dim=-1) for ts in zip(*out)] else: out = torch.stack(out, dim=-1) return out class ModelWrapper: """ Wrapper created to ease the training/testing/loading. Args:
# MIT licensed # Copyright (c) 2019-2020 lilydjwg <*****@*****.**>, et al. import structlog from typing import Optional, Dict, Mapping import json as _json from ..ctxvars import tries, proxy, user_agent logger = structlog.get_logger(logger_name=__name__) class Response: '''The response of an HTTP request. .. py:attribute:: body :type: bytes .. py:attribute:: headers :type: Mapping[str, str] ''' def __init__( self, headers: Mapping[str, str], body: bytes, ) -> None: self.headers = headers self.body = body def json(self): '''Convert reponse content to JSON.''' return _json.loads(self.body.decode('utf-8'))
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import re from structlog import get_logger from twisted.internet import reactor from twisted.internet.base import DelayedCall from twisted.internet.defer import inlineCallbacks, returnValue, Deferred from simplejson import dumps, loads from common.utils.asleep import asleep log = get_logger() class Worker(object): """ Worker side of the coordinator. An instance of this class runs in every voltha instance. It monitors what work is assigned to this instance by the leader. This is all done via consul. """ ASSIGNMENT_EXTRACTOR = '^%s(?P<member_id>[^/]+)/(?P<work_id>[^/]+)$' # Public methods: def __init__(self, instance_id, coordinator):
import traceback from asyncio import Future from collections import OrderedDict, namedtuple from contextlib import suppress from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Generic, Optional, Tuple, TypeVar, Union, cast from structlog import get_logger from txstratum.constants import DEFAULT_EXPECTED_MINING_TIME if TYPE_CHECKING: from asyncio.events import AbstractEventLoop from hathorlib import BaseTransaction logger = get_logger() class Periodic: """Create an asyncio task that calls an async function periodically. The function is called every `T` seconds, not taking into consideration how long the last call took. If the duration of a call is longer than `T` seconds, the function will be called immediately after it finishes. Adapted from: - https://stackoverflow.com/a/37514633/947511 - https://stackoverflow.com/a/55505152/947511 """ def __init__(self, afunc: Callable[..., Awaitable[None]],
# -*- coding: utf-8 -*- from typing import Optional from reprlib import repr as _repr from time import perf_counter as perf import structlog from ..typedefs import HTTPRequest from ..typedefs import HTTPResponse from ..errors import JsonRpcError logger = structlog.get_logger('jussi') async def initialize_jussi_request( request: HTTPRequest) -> Optional[HTTPResponse]: # parse jsonrpc try: request.jsonrpc except JsonRpcError as e: return e.to_sanic_response() except Exception as e: return JsonRpcError(http_request=request, exception=e).to_sanic_response() async def finalize_jussi_response(request: HTTPRequest, response: HTTPResponse) -> None: # pylint: disable=bare-except try: response.headers['x-jussi-request-id'] = request.jussi_request_id
def create_app(cmdline=False, test_config=None): app = Flask(__name__) relengapi_logging.configure_logging(app) logger = structlog.get_logger() env_var = 'RELENGAPI_SETTINGS' if test_config: app.config.update(**test_config) else: if env_var in os.environ and os.environ[env_var]: app.config.from_envvar(env_var) else: logger.warning( "Using default settings; to configure relengapi, set " "%s to point to your settings file" % env_var) # reconfigure logging now that we have loaded configuration relengapi_logging.configure_logging(app) # and re-construct the logger to get updated configuration logger = structlog.get_logger() # add the necessary components to the app app.db = db.make_db(app) app.celery = celery.make_celery(app) layout.init_app(app) auth.init_app(app) api.init_app(app) aws.init_app(app) memcached.init_app(app) app.relengapi_blueprints = {} for bp in blueprints: if cmdline: logger.info("registering blueprint %s", bp.name) app.register_blueprint(bp, url_prefix='/%s' % bp.name) app.relengapi_blueprints[bp.name] = bp # set up a random session key if none is specified if not app.config.get('SECRET_KEY'): logger.warning( "setting per-process session key - sessions will be reset on " "process restart") app.secret_key = os.urandom(24) request_id_header = app.config.get('REQUEST_ID_HEADER') def get_req_id_uuid(): return str(uuid.uuid4()) def get_req_id_header(): return request.headers.get(request_id_header) or get_req_id_uuid() get_req_id = get_req_id_header if request_id_header else get_req_id_uuid @app.before_request def setup_request(): # set up `g` g.db = app.db g.request_id = get_req_id() # reset the logging context, deleting any info for the previous request # in this thread and binding new relengapi_logging.reset_context(request_id=g.request_id, user=str(current_user)) @app.route('/') def root(): # render all of the blueprints' templates first bp_widgets = [] for bp in app.blueprints.itervalues(): bp_widgets.extend(bp.root_widget_templates or []) bp_widgets.sort() bp_widgets = [ tpl for (_, tpl, condition) in bp_widgets if not condition or condition() ] return render_template('root.html', bp_widgets=bp_widgets) @app.route('/versions') @api.apimethod(VersionInfo) def versions(): dists = {} for dist in introspection.get_distributions().itervalues(): dists[dist.key] = DistributionInfo(project_name=dist.project_name, version=dist.version, relengapi_metadata={}) blueprints = {} relengapi_dist = pkg_resources.get_distribution('relengapi') for bp in app.relengapi_blueprints.itervalues(): blueprints[bp.name] = BlueprintInfo(distribution='relengapi', version=relengapi_dist.version) return VersionInfo(distributions=dists, blueprints=blueprints) return app
# -*- encoding: utf-8 -*- """Metrics for an evergreen version.""" from __future__ import absolute_import, division from datetime import datetime from typing import Callable, Dict, List, Optional, TYPE_CHECKING from structlog import get_logger if TYPE_CHECKING: from evergreen.build import Build from evergreen.version import Version from evergreen.metrics.buildmetrics import BuildMetrics LOGGER = get_logger(__name__) class VersionMetrics(object): """Metrics about an evergreen version.""" def __init__(self, version: "Version") -> None: """ Create an instance of version metrics. :param version: Version to analyze. """ self.version = version self.total_processing_time = 0 self.task_success_count = 0 self.task_failure_count = 0
import os import datetime import logging import structlog import sys import time import configparser import re from pkg_resources import resource_filename from cpo_pipeline.drmaa import prepare_job, run_jobs from cpo_pipeline.logging import now from cpo_pipeline.resistance.parsers import result_parsers logger = structlog.get_logger() def main(args): """ main entrypoint Args: args(argparse.Namespace): Parsed command-line arguments. Returns: (void) """ config = configparser.ConfigParser() config.read(args.config_file) sample_id = args.sample_id
def __init__(self): """Initializes StrategyAnalyzer class """ self.logger = structlog.get_logger()
class BlockFetcher: """Fetches new blocks via a web3 interface and passes them on to a set of callbacks.""" logger = structlog.get_logger("monitor.block_fetcher") def __init__( self, state, w3, db, max_reorg_depth=1000, initial_block_resolver=None ): self.w3 = w3 self.db = db self.max_reorg_depth = max_reorg_depth self.head = state.head self.current_branch = state.current_branch self.report_callbacks = [] self.initial_block_resolver = initial_block_resolver self.initial_blocknr = state.initial_blocknr self._start_sync_number = 0 self.syncing = False @classmethod def from_fresh_state(cls, *args, **kwargs): return cls(cls.get_fresh_state(), *args, **kwargs) @classmethod def get_fresh_state(cls): return BlockFetcherStateV1(head=None, current_branch=[], initial_blocknr=0) @property def state(self): return BlockFetcherStateV1( head=self.head, current_branch=self.current_branch, initial_blocknr=self.initial_blocknr, ) @property def _backwards_sync_in_progress(self) -> bool: return len(self.current_branch) > 0 def register_report_callback(self, callback): self.report_callbacks.append(callback) def _run_callbacks(self, blocks): for block in blocks: for callback in self.report_callbacks: callback(block) def _insert_branch(self, blocks): if len(blocks) == 0: return if blocks[0].number not in (0, self.initial_blocknr) and not self.db.contains( blocks[0].parentHash ): raise FetchingForkWithUnkownBaseError( "Tried to insert branch from a fork with unknown parent block." ) try: self.db.insert_branch(blocks) self.head = blocks[-1] self.current_branch.clear() except AlreadyExists: raise ValueError("Tried to insert already known block") self._run_callbacks(blocks) def _insert_first_block(self): resolver = self.initial_block_resolver or blocksel.ResolveGenesisBlock() block = resolver.resolve_block(self.w3) if not block: raise ValueError("Can't fetch initial block to sync from!") latest = self.w3.eth.getBlock("latest") safe_initial_blocknr = max(latest.number - self.max_reorg_depth, 0) if block.number > safe_initial_blocknr: unsafe_block = block block = self.w3.eth.getBlock(safe_initial_blocknr) self.logger.warn( f"choosing {format_block(block)} instead of {format_block(unsafe_block)}" ) self.initial_blocknr = block.number self.logger.info( f"starting initial sync from {format_block(block)}, latest {format_block(latest)}" ) self._insert_branch([block]) def fetch_and_insert_new_blocks( self, *, max_number_of_blocks=5000, max_block_height: int = None ): """Fetches up to `max_number_of_blocks` blocks and only up to blocknumber `max_block_height` (inclusive) and updates the internal state If a full branch is fetched it also inserts the new blocks Returns the number of fetched blocks """ if max_number_of_blocks < 1: return 0 number_of_synced_blocks = 0 if self.db.is_empty(): self._insert_first_block() number_of_synced_blocks += 1 self._save_sync_start() if not self._backwards_sync_in_progress: forward_sync_target = self.fetch_forward_sync_target() # sync forwards at most up until the forward sync target, but no more than # max_number_of_blocks max_forward_block_height = ( forward_sync_target if max_block_height is None else min(forward_sync_target, max_block_height) ) max_forward_sync_blocks = max( 0, max_number_of_blocks - number_of_synced_blocks ) number_of_synced_blocks += self._sync_forwards( max_number_of_blocks=max_forward_sync_blocks, max_block_height=max_forward_block_height, ) # sync backwards until we have synced max_number_of_blocks in total or we are fully synced assert 0 <= number_of_synced_blocks <= max_number_of_blocks max_backward_sync_blocks = max_number_of_blocks - number_of_synced_blocks number_of_synced_blocks += self._sync_backwards( max_number_of_blocks=max_backward_sync_blocks, max_block_height=max_block_height, ) return number_of_synced_blocks def fetch_forward_sync_target(self): return max(self.w3.eth.blockNumber - self.max_reorg_depth, 0) def _sync_forwards( self, *, max_number_of_blocks: int, max_block_height: int ) -> int: block_numbers_to_fetch = range( self.head.number + 1, min(self.head.number + 1 + max_number_of_blocks, max_block_height + 1), ) blocks = list( itertools.takewhile( lambda block: block is not None, ( self.w3.eth.getBlock(block_number) for block_number in block_numbers_to_fetch ), ) ) self._insert_branch(blocks) return len(blocks) def _sync_backwards( self, *, max_number_of_blocks: int, max_block_height: int = None ) -> int: branch_length_before = len(self.current_branch) complete = self._fetch_branch( max_number_of_blocks, head_block_id=max_block_height ) number_of_fetched_blocks = len(self.current_branch) - branch_length_before if complete and len(self.current_branch) > 0: self._insert_branch(list(reversed(self.current_branch))) return number_of_fetched_blocks def _get_block(self, block_id): """call self.w3.eth.getBlock, but make sure we don't fetch a block before the initial block""" block = self.w3.eth.getBlock(block_id) assert block is not None, f"Could not fetch block {block_id}" if block.number < self.initial_blocknr: self.logger.error( f"Fetched block with number {block.number} < {self.initial_blocknr} (initial block number) on syncing backwards!" ) raise FetchingForkWithUnkownBaseError( "Synchronized backwards on a fork with base before initial synchronized block!" ) return block def _fetch_branch(self, max_blocks_to_fetch, head_block_id=None): """ Starts or continues to fetch a branch :param max_blocks_to_fetch: Max number of blocks to fetch before return :param head_block_id: head block id of the branch to fetch, defaults to 'latest' :return: True if the full branch was fetched, False if it needs to continue on the next call """ if max_blocks_to_fetch < 0: raise ValueError("Maximum number of blocks to fetch must not be negative") elif max_blocks_to_fetch == 0: return False number_of_fetched_blocks = 0 if len(self.current_branch) == 0: if head_block_id is None: head_block_id = "latest" head = self._get_block(head_block_id) if self.db.contains(head.hash): self.logger.debug( "no new blocks", head_hash=self.head.hash, head_number=self.head.number, ) return True self.current_branch = [head] number_of_fetched_blocks += 1 while ( not number_of_fetched_blocks >= max_blocks_to_fetch and not self.db.contains(self.current_branch[-1].parentHash) ): parent = self._get_block(self.current_branch[-1].parentHash) self.current_branch.append(parent) number_of_fetched_blocks += 1 complete = self.db.contains(self.current_branch[-1].parentHash) return complete def get_sync_status(self): last_block_number = self.w3.eth.blockNumber head_block_number = self.head_block_number if last_block_number <= self._start_sync_number: return 0 # limit it to not go over 100 % branch_correction = min( len(self.current_branch), last_block_number - head_block_number ) return (head_block_number - self._start_sync_number + branch_correction) / ( last_block_number - self._start_sync_number ) def _save_sync_start(self): # To show sync status, remember start sync block if not self.syncing and self.head.number < self.w3.eth.blockNumber - 5: self._start_sync_number = self.head.number self.syncing = True if self.syncing and self.head.number >= self.w3.eth.blockNumber - 1: self.syncing = False @property def head_block_number(self): if self.head is None: return 0 return self.head.number
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging.config import threading import uuid import structlog request_logger = structlog.get_logger("warehouse.request") RENDERER = structlog.processors.JSONRenderer() class StructlogFormatter(logging.Formatter): def format(self, record): # TODO: Figure out a better way of handling this besides just looking # at the logger name, ideally this would have some way to # really differentiate between log items which were logged by # structlog and which were not. if not record.name.startswith("warehouse."): # TODO: Is there a better way to handle this? Maybe we can figure # out a way to pass this through the structlog processors # instead of manually duplicating the side effects here? event_dict = {
from __future__ import print_function import random from sortedcontainers import SortedDict import structlog from raidex.utils import pex from raidex.utils.timestamp import to_str_repr from raidex.raidex_node.order.offer import OfferType from eth_utils import int_to_big_endian log = structlog.get_logger('node.offer_book') def generate_random_offer_id(): # generate random offer-id in the 32byte int range return int(random.randint(0, 2**256 - 1)) class OfferDeprecated(object): """ Represents an Offer from the Broadcast. the broadcasted offer_message stores absolute information (bid_token, ask_token, bid_amount, ask_amount) the Offer stores it's information relative to it's market (type, price) Internally we work with relative values because: 1) we want to easily compare prices (prices are the ultimate ordering criterion) 2) we want to separate BUYs from SELLs 3) traders are used to this!
from requests.exceptions import RequestException from readthedocs.api.v2.client import api from readthedocs.builds import utils as build_utils from readthedocs.builds.constants import ( BUILD_STATUS_SUCCESS, SELECT_BUILD_STATUS, ) from readthedocs.core.permissions import AdminPermission from readthedocs.integrations.models import Integration from ..constants import GITHUB from ..models import RemoteOrganization, RemoteRepository from .base import Service, SyncServiceError log = structlog.get_logger(__name__) class GitHubService(Service): """Provider service for GitHub.""" adapter = GitHubOAuth2Adapter # TODO replace this with a less naive check url_pattern = re.compile(r'github\.com') vcs_provider_slug = GITHUB def sync_repositories(self): """Sync repositories from GitHub API.""" remote_repositories = [] try:
from structlog import get_logger from structlog.testing import capture_logs with capture_logs() as cap_logs: get_logger().bind(x="y").info("hello") cap_logs [{"x": "y", "event": "hello", "log_level": "info"}]
Methods for clustering point collections. Methods available are designed to work with infrastructure elements, but can be used to any other point collection. """ import structlog from typing import List from ...core.query import Query from ...core.mixins import GeoDataMixin from .versioned_infrastructure import VersionedInfrastructure logger = structlog.get_logger("flowmachine.debug", submodule=__name__) class LocationCluster(GeoDataMixin, Query): """ Class for computing clusters of points using different algorithms. This class was designed to work with infrastructure elements (i.e. towers/sites), but can also be used with other point collection as long as that is a table in the database. This class currently implements three methods: K-means, DBSCAN, and Area. K-means is a clustering algorithm that clusters together points based on the point's distance to a point representing the centroid of the cluster. The algorithm has two steps: (a) point allocation and (b) centroid re-calculation. In (a) it allocates
import kombu.exceptions import redis.exceptions from clickhouse_driver.errors import Error as ClickhouseError from django.core.cache import cache from django.db import DEFAULT_DB_ALIAS from django.db import Error as DjangoDatabaseError from django.db import connections from django.db.migrations.executor import MigrationExecutor from django.http import HttpRequest, HttpResponse, JsonResponse from structlog import get_logger from ee.clickhouse.client import sync_execute from ee.kafka_client.client import can_connect as can_connect_to_kafka from posthog.celery import app logger = get_logger(__file__) ServiceRole = Literal["events", "web", "worker"] service_dependencies: Dict[ServiceRole, List[str]] = { "events": ["http", "kafka_connected"], "web": [ "http", "postgres", "postgres_migrations_uptodate", "cache", # NOTE: we do not include clickhouse for web, as even without clickhouse we # want to be able to display something to the user. # "clickhouse" # NOTE: we do not include "celery_broker" as web could still do lot's of # useful things
class LightClientMessageHandler: log = structlog.get_logger(__name__) # pylint: disable=invalid-name @classmethod def store_light_client_protocol_messages(cls, messages: List[Message], wal: WriteAheadLog): protocol_messages = list( map(build_light_client_protocol_message, messages)) assert len(messages) == len( protocol_messages), "Light client protocol message persist error" to_store = [] for msg_dto in protocol_messages: to_store.append(DbLightClientProtocolMessage(msg_dto)) return wal.storage.write_light_client_protocol_messages(to_store) @classmethod def store_light_client_protocol_message(cls, identifier: int, message: Message, signed: bool, payment_id: int, order: int, wal: WriteAheadLog): return wal.storage.write_light_client_protocol_message( message, build_light_client_protocol_message(identifier, message, signed, payment_id, order)) @classmethod def store_received_locked_transfer(cls, identifier: int, message: Message, signed: bool, payment_id: int, order: int, storage: SerializedSQLiteStorage): return storage.write_light_client_protocol_message( message, build_light_client_protocol_message(identifier, message, signed, payment_id, order)) @classmethod def update_stored_msg_set_signed_data(cls, message: Message, payment_id: int, order: int, wal: WriteAheadLog): return wal.storage.update_light_client_protocol_message_set_signed_data( payment_id, order, message) @classmethod def store_light_client_payment(cls, payment: LightClientPayment, storage: SerializedSQLiteStorage): return storage.write_light_client_payment(payment) @classmethod def is_light_client_protocol_message_already_stored( cls, payment_id: int, order: int, wal: WriteAheadLog): existing_message = wal.storage.is_light_client_protocol_message_already_stored( payment_id, order) if existing_message: return LightClientProtocolMessage(existing_message[4] is not None, existing_message[3], existing_message[2], existing_message[1], existing_message[4], existing_message[5]) return existing_message @classmethod def is_light_client_protocol_message_already_stored_message_id( cls, message_id: int, payment_id: int, order: int, wal: WriteAheadLog): return wal.storage.is_light_client_protocol_message_already_stored_with_message_id( message_id, payment_id, order) @classmethod def get_light_client_protocol_message_by_identifier( cls, message_identifier: int, wal: WriteAheadLog): message = wal.storage.get_light_client_protocol_message_by_identifier( message_identifier) return LightClientProtocolMessage(message[3] is not None, message[1], message[4], message[0], message[2], message[3]) @classmethod def get_light_client_payment_locked_transfer(cls, payment_identifier: int, wal: WriteAheadLog): message = wal.storage.get_light_client_payment_locked_transfer( payment_identifier) return LightClientProtocolMessage(message[3] is not None, message[1], message[4], message[0], message[2], message[3]) @staticmethod def get_order_for_ack(ack_parent_type: string, ack_type: string, is_delivered_from_initiator: bool = False): switcher_processed = { LockedTransfer.__name__: 3, Secret.__name__: 13, } switcher_delivered = { LockedTransfer.__name__: 4 if is_delivered_from_initiator else 2, RevealSecret.__name__: 10 if is_delivered_from_initiator else 8, SecretRequest.__name__: 6, Secret.__name__: 14 if is_delivered_from_initiator else 12, } if ack_type.lower() == "processed": return switcher_processed.get(ack_parent_type, -1) else: return switcher_delivered.get(ack_parent_type, -1) @classmethod def exists_payment(cls, payment_id: int, wal: WriteAheadLog): return wal.storage.exists_payment(payment_id) @classmethod def store_lc_processed(cls, message: Processed, wal: WriteAheadLog): # If exists for that payment, the same message by the order, then discard it. message_identifier = message.message_identifier # get first principal message by message identifier protocol_message = LightClientMessageHandler.get_light_client_protocol_message_by_identifier( message_identifier, wal) json_message = None if protocol_message.signed_message is None: json_message = protocol_message.unsigned_message else: json_message = protocol_message.signed_message json_message = json.loads(json_message) order = LightClientMessageHandler.get_order_for_ack( json_message["type"], message.__class__.__name__.lower()) if order == -1: cls.log.error( "Unable to find principal message for {} {}: ".format( message.__class__.__name__, message_identifier)) else: exists = LightClientMessageHandler.is_light_client_protocol_message_already_stored_message_id( message_identifier, protocol_message.light_client_payment_id, order, wal) if not exists: LightClientMessageHandler.store_light_client_protocol_message( message_identifier, message, True, protocol_message.light_client_payment_id, order, wal) else: cls.log.info( "Message for lc already received, ignoring db storage") @classmethod def store_lc_delivered(cls, message: Delivered, wal: WriteAheadLog): # If exists for that payment, the same message by the order, then discard it. message_identifier = message.delivered_message_identifier # get first by message identifier protocol_message = LightClientMessageHandler.get_light_client_protocol_message_by_identifier( message_identifier, wal) json_message = None if protocol_message.signed_message is None: json_message = protocol_message.unsigned_message else: json_message = protocol_message.signed_message json_message = json.loads(json_message) first_message_is_lt = protocol_message.message_order == 1 is_delivered_from_initiator = True delivered_sender = message.sender if not first_message_is_lt: # get lt to get the payment identifier locked_transfer = LightClientMessageHandler.get_light_client_payment_locked_transfer( protocol_message.light_client_payment_id, wal) signed_locked_transfer_message = json.loads( locked_transfer.signed_message) payment_initiator = signed_locked_transfer_message["initiator"] if to_checksum_address(delivered_sender) != to_checksum_address( payment_initiator): is_delivered_from_initiator = False else: # message is the lt payment_initiator = json_message["initiator"] if to_checksum_address(delivered_sender) != to_checksum_address( payment_initiator): is_delivered_from_initiator = False order = LightClientMessageHandler.get_order_for_ack( json_message["type"], message.__class__.__name__.lower(), is_delivered_from_initiator) if order == -1: cls.log.error( "Unable to find principal message for {} {}: ".format( message.__class__.__name__, message_identifier)) else: exists = LightClientMessageHandler.is_light_client_protocol_message_already_stored_message_id( message_identifier, protocol_message.light_client_payment_id, order, wal) if not exists: LightClientMessageHandler.store_light_client_protocol_message( message_identifier, message, True, protocol_message.light_client_payment_id, order, wal) else: cls.log.info( "Message for lc already received, ignoring db storage") @classmethod def store_update_non_closing_balance_proof( cls, non_closing_balance_proof_data: LightClientNonClosingBalanceProof, storage: SerializedSQLiteStorage): return storage.write_light_client_non_closing_balance_proof( non_closing_balance_proof_data) @classmethod def get_latest_light_client_non_closing_balance_proof( cls, channel_id: int, storage: SerializedSQLiteStorage): latest_update_balance_proof_data = storage.get_latest_light_client_non_closing_balance_proof( channel_id) if latest_update_balance_proof_data: balance_proof = Unlock.from_dict( json.loads(latest_update_balance_proof_data[7])) return LightClientNonClosingBalanceProof( latest_update_balance_proof_data[1], latest_update_balance_proof_data[2], latest_update_balance_proof_data[3], latest_update_balance_proof_data[4], latest_update_balance_proof_data[5], latest_update_balance_proof_data[6], balance_proof, latest_update_balance_proof_data[8], latest_update_balance_proof_data[0]) return None
""" teatro - get_ranking """ import structlog from telegram import ChatAction from telegram.ext.dispatcher import run_async from api import parse_alternativa logger = structlog.get_logger(filename=__name__) @run_async def get_ranking(update, context, *args, **kwargs): context.bot.send_chat_action(chat_id=update.message.chat_id, action=ChatAction.TYPING) logger.info(f"Teatro... by {update.message.from_user.name}") data = parse_alternativa() if not data: return if not context.args: text = "\n".join(data[:5]) text = ( f"Ranking de lo más buscado en teatro:\n{text}\nBy http://www.alternativateatral.com/" ) context.bot.send_message(chat_id=update.message.chat_id, text=text) return try:
def __init__(self, technology, extra_args, device_id, backend, host, port): """ Create PONResourceManager object. :param technology: PON technology :param: extra_args: This string contains extra arguments passed during pre-provisioning of OLT and specifies the OLT Vendor type :param device_id: OLT device id :param backend: backend store :param host: ip of backend store :param port: port on which backend store listens :raises exception when invalid backend store passed as an argument """ # logger self._log = structlog.get_logger() try: self.technology = technology self.extra_args = extra_args self.device_id = device_id self.backend = backend self.host = host self.port = port self.olt_model = None self._kv_store = ResourceKvStore(technology, device_id, backend, host, port) self.tech_profile = TechProfile(self) # Below attribute, pon_resource_ranges, should be initialized # by reading from KV store. self.pon_resource_ranges = dict() self.pon_resource_ranges[ PONResourceManager.ONU_ID_SHARED_IDX] = None self.pon_resource_ranges[ PONResourceManager.ALLOC_ID_SHARED_IDX] = None self.pon_resource_ranges[ PONResourceManager.GEMPORT_ID_SHARED_IDX] = None self.pon_resource_ranges[ PONResourceManager.FLOW_ID_SHARED_IDX] = None self.shared_resource_mgrs = dict() self.shared_resource_mgrs[ PONResourceManager.ONU_ID_SHARED_IDX] = None self.shared_resource_mgrs[ PONResourceManager.ALLOC_ID_SHARED_IDX] = None self.shared_resource_mgrs[ PONResourceManager.GEMPORT_ID_SHARED_IDX] = None self.shared_resource_mgrs[ PONResourceManager.FLOW_ID_SHARED_IDX] = None self.shared_idx_by_type = dict() self.shared_idx_by_type[ PONResourceManager. ONU_ID] = PONResourceManager.ONU_ID_SHARED_IDX self.shared_idx_by_type[ PONResourceManager. ALLOC_ID] = PONResourceManager.ALLOC_ID_SHARED_IDX self.shared_idx_by_type[ PONResourceManager. GEMPORT_ID] = PONResourceManager.GEMPORT_ID_SHARED_IDX self.shared_idx_by_type[ PONResourceManager. FLOW_ID] = PONResourceManager.FLOW_ID_SHARED_IDX self.intf_ids = None except Exception as e: self._log.exception("exception-in-init") raise Exception(e)
def __init__(self): """Initializes IchimokuCloud class """ self.logger = structlog.get_logger()
import structlog from raidex import messages from raidex.utils import pex from raidex.commitment_service.swap import SwapFactory from raidex.raidex_node.listener_tasks import ListenerTask from raidex.trader_mock.trader import TransferReceipt from raidex.trader_mock.trader import TransferReceivedListener from raidex.message_broker.listeners import ( TakerCommitmentListener, CommitmentListener, SwapExecutionListener, CancellationListener, ) log = structlog.get_logger('commitment_service') log_swaps = structlog.get_logger('commitment_service.asset_swaps') log_messaging = structlog.get_logger('commitment_service.messaging') log_refunds = structlog.get_logger('commitment_service.refunds') log_trader = structlog.get_logger('commitment_service.trader') class QueueListenerTask(gevent.Greenlet): def __init__(self, queue): self.queue = queue gevent.Greenlet.__init__(self) def _run(self): while True: data = self.queue.get() self.process(data)
from raiden.transfer.state_change import ActionChangeNodeNetworkState from raiden.utils import pex from raiden.utils.notifying_queue import NotifyingQueue from raiden.utils.runnable import Runnable from raiden.utils.typing import ( MYPY_ANNOTATION, Address, Dict, List, MessageID, Nonce, Tuple, UDPMessageID, ) log = structlog.get_logger(__name__) # pylint: disable=invalid-name log_healthcheck = structlog.get_logger(__name__ + ".healthcheck") # pylint: disable=invalid-name QueueItem_T = Tuple[bytes, MessageID] Queue_T = List[QueueItem_T] # GOALS: # - Each netting channel must have the messages processed in-order, the # transport must detect unacknowledged messages and retry them. # - A queue must not stall because of synchronization problems in other queues. # - Assuming a queue can stall, the unhealthiness of a node must not be # inferred from the lack of acknowledgement from a single queue, but healthiness # may be safely inferred from it. # - The state of the node must be synchronized among all tasks that are # handling messages.
import structlog import time from csv import DictReader from adjuntos.models import Attachment from elecciones.models import Categoria, Circuito, MesaCategoria from elecciones.management.commands.basic_command import BaseCommand from scheduling.scheduler import scheduler from scheduling.models import ColaCargasPendientes logger = structlog.get_logger('scheduler') class Command(BaseCommand): help = "Toma una lista de circuitos y la prioriza." def priorizar_circuito(self, nuevas, lugar_en_cola, linea, slug_cat, nro_distrito, nro_seccion, nro_circuito, cant_mesas_necesarias): try: categoria = Categoria.objects.get(slug=slug_cat) except Circuito.DoesNotExist: logger.error(f"No existe la categoría con slug {slug_cat} (línea {linea}).") return lugar_en_cola try: circuito = Circuito.objects.get( numero=nro_circuito, seccion__numero=nro_seccion, seccion__distrito__numero=nro_distrito ) except Circuito.DoesNotExist: logger.error(f"No existe el circuito nro {nro_circuito} en la sección nro {nro_seccion} y distrito {nro_distrito} (línea {linea}).")