def get_django_wsgi(settings_module): xray_recorder.begin_subsegment('django_init') from django.core.wsgi import get_wsgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_module) import django if django.VERSION[0] <= 1 and django.VERSION[1] < 7: # call django.setup only for django <1.7.0 # (because setup already in get_wsgi_application since that) # https://github.com/django/django/commit/80d74097b4bd7186ad99b6d41d0ed90347a39b21 django.setup() xray_recorder.end_subsegment() xray_recorder.begin_subsegment('patch_libs') patch(('psycopg2', 'requests')) xray_recorder.end_subsegment() xray_recorder.begin_subsegment('application_init') application = get_wsgi_application() xray_recorder.end_subsegment() return application
def ready(self): try: # noinspection PyPackageRequirements import users.signals # noqa F401 except ImportError: pass patch(('requests',))
def initialize(cls): context = XrayContext() xray_recorder.configure( emitter=cls.use_daemon is False and cls.emitter or None, context=context, sampling=True, context_missing='LOG_ERROR') patch(['boto3', 'requests']) logging.getLogger('aws_xray_sdk.core').setLevel(logging.ERROR)
def initialize(cls): context = XrayContext() xray_recorder.configure(emitter=cls.use_daemon is False and cls.emitter or None, context=context, sampling=True, context_missing='LOG_ERROR') patch(['boto3', 'requests']) logging.getLogger('aws_xray_sdk.core').setLevel(logging.ERROR)
def create_app(script_info=None): # instantiate the app app = Flask(__name__) flask_bcrypt = Bcrypt(app) jwt = JWTManager(app) app.json_encoder = JSONEncoder # enable CORS CORS(app, resources={r'/*': {'origins': '*'}}) # TODO 9: Review X-ray setting patch_modules = ( 'boto3', 'botocore', 'pynamodb', 'requests', ) plugins = ('EC2Plugin', ) xray_recorder.configure(service='CloudAlbum', plugins=plugins, context_missing='LOG_ERROR', sampling=False) xray_recorder.begin_segment('cloudalbum') XRayMiddleware(app, xray_recorder) patch(patch_modules) # set config app_settings = os.getenv('APP_SETTINGS') app.config.from_object(app_settings) # set logger to STDOUT app.logger.addHandler(logging.StreamHandler(sys.stdout)) app.logger.setLevel(logging.DEBUG) # Create database table, if it is not exists with app.app_context(): create_table() # register blueprints from cloudalbum.api.users import users_blueprint app.register_blueprint(users_blueprint, url_prefix='/users') from cloudalbum.api.photos import photos_blueprint app.register_blueprint(photos_blueprint, url_prefix='/photos') from cloudalbum.api.admin import admin_blueprint app.register_blueprint(admin_blueprint, url_prefix='/admin') # shell context for flask cli @app.shell_context_processor def ctx(): return {'app': app} return app
def construct_ctx(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ patch(('pg8000', )) xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities() unpatch()
def init_xray(app): logging.getLogger("aws_xray_sdk").setLevel('ERROR') if app.debug: # env var AWS_XRAY_SDK_ENABLED can overwrite this global_sdk_config.set_sdk_enabled(False) else: # TODO: configure x-ray service xray_recorder.configure(service="btbapi") # Setup X-Ray Flask Integration XRayMiddleware(app, xray_recorder) # Setup X-Ray psycopg2, boto3 (aws sdk) Integration patch(["psycopg2", "boto3"])
def test_db(self): patch(('sqlite3', )) url = reverse('call_db') self.client.get(url) segment = xray_recorder.emitter.pop() assert len(segment.subsegments) == 1 subsegment = segment.subsegments[0] assert subsegment.name == ':memory:' assert not subsegment.in_progress sql = subsegment.sql assert sql['database_type'] == 'sqlite3' assert sql['database_version']
def ready(self): """ Configure global XRay recorder based on django settings under XRAY_RECORDER namespace. This method could be called twice during server startup because of base command and reload command. So this function must be idempotent """ if not settings.AWS_XRAY_TRACING_NAME: raise SegmentNameMissingException('Segment name is required.') xray_recorder.configure( daemon_address=settings.AWS_XRAY_DAEMON_ADDRESS, sampling=settings.SAMPLING, sampling_rules=settings.SAMPLING_RULES, sampler=settings.SAMPLER, context_missing=settings.AWS_XRAY_CONTEXT_MISSING, plugins=settings.PLUGINS, service=settings.AWS_XRAY_TRACING_NAME, dynamic_naming=settings.DYNAMIC_NAMING, streaming_threshold=settings.STREAMING_THRESHOLD, max_trace_back=settings.MAX_TRACE_BACK, stream_sql=settings.STREAM_SQL, ) if settings.PATCH_MODULES: if settings.AUTO_PATCH_PARENT_SEGMENT_NAME is not None: with xray_recorder.in_segment( settings.AUTO_PATCH_PARENT_SEGMENT_NAME): patch( settings.PATCH_MODULES, ignore_module_patterns=settings.IGNORE_MODULE_PATTERNS) else: patch(settings.PATCH_MODULES, ignore_module_patterns=settings.IGNORE_MODULE_PATTERNS) # if turned on subsegment will be generated on # built-in database and template rendering if settings.AUTO_INSTRUMENT: try: patch_db() except Exception: log.debug('failed to patch Django built-in database') try: patch_template() except Exception: log.debug('failed to patch Django built-in template engine')
def construct_ctx(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ from aws_xray_sdk.ext.httplib import unpatch, reset_ignored patch(('httplib',)) xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities() unpatch() reset_ignored()
def engine(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ from aws_xray_sdk.ext.sqlalchemy_core import unpatch patch(('sqlalchemy_core',)) engine = create_engine('sqlite:///:memory:') xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.begin_segment('name') Base.metadata.create_all(engine) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield engine xray_recorder.clear_trace_entities() unpatch()
def init_app(cls, app: Insanic, recorder: AsyncAWSXRayRecorder = None) -> None: """ Initializes Insanic to use Incendiary. - This loads all default Incendiary configs. - Validates connection information to X-Ray Daemon. - Configures X-Ray SDK Recorder - Attaches middlewares to start stop segments - Replaces :code:`Service` object with :code:`IncendiaryService` to trace interservice communications. - Replaces asyncio task factory. - Patches configured modules. :param app: Your Insanic application/ :param recorder: If you want to use your own recorder. """ # checks to see if tracing can be enabled cls.app = app cls.load_config(app.config) messages = cls._check_prerequisites(app) if len(messages) == 0: global_sdk_config.set_sdk_enabled(True) app.xray_recorder = recorder or xray_recorder cls.setup_middlewares(app) cls.setup_client(app) cls.setup_listeners(app) patch(app.config.INCENDIARY_XRAY_PATCH_MODULES, raise_errors=False) app.plugin_initialized("incendiary", cls) else: cls._handle_error(app, messages) app.config.INCENDIARY_XRAY_ENABLED = False global_sdk_config.set_sdk_enabled(False)
import hashlib import json import logging import os from datetime import datetime from decimal import Decimal from json.decoder import JSONDecodeError import boto3 from aws_xray_sdk.core import patch # type: ignore from aws_xray_sdk.core import xray_recorder # type: ignore from clparser import parse_request_body, parse_page, PostRemovedException, CL404Exception # x-ray tracing patch(['boto3', 'botocore']) LOGGER = logging.getLogger() if os.environ.get("LOG_LEVEL", "INFO") == "DEBUG": LOGGER.setLevel(logging.DEBUG) else: LOGGER.setLevel(logging.INFO) # Quick sanity checks and predefined local dev if os.getenv("AWS_SAM_LOCAL", ""): DYNAMO = boto3.resource('dynamodb', endpoint_url="http://dynamodb:8000") else: DYNAMO = boto3.resource('dynamodb') TABLE = DYNAMO.Table(os.getenv("TABLE_NAME", "apthunt")) # Create SQS client
api.add_resource(HealthCheck, '/healthcheck', '/') api.add_resource(Unicorn, '/unicorn') api.add_resource(Unicorns, '/unicorn/<string:unicorn_id>') if __name__ == '__main__': #If running in prod - log to xray and CWL try: import watchtower handler = watchtower.CloudWatchLogHandler(log_group='CICDApiProxy', ) app.logger.addHandler(handler) logging.getLogger("werkzeug").addHandler(handler) logging.getLogger('aws_xray_sdk').addHandler(handler) logging.getLogger('requests').addHandler(handler) except: print "Couldn't start CW Logging" #Lets try to use AWS X-ray for metrics / logging if available to us try: from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.ext.flask.middleware import XRayMiddleware xray_recorder.configure(service='CICDApiProxy') xray_recorder.configure(context_missing='LOG_ERROR') XRayMiddleware(app, xray_recorder) from aws_xray_sdk.core import patch patch(('requests')) except: print 'Failed to import X-ray' app.run(host='0.0.0.0')
from collections import namedtuple import urllib.request, urllib.error, urllib.parse from retrying import retry import requests import pickle import hashlib import json import itertools requests.packages.urllib3.disable_warnings() from aws_xray_sdk.core import patch if hasattr(settings, 'XRAY_RECORDER'): patch(('requests', )) SOLR_DEFAULTS = { 'mm': '100%', 'pf3': 'title', 'pf': 'text,title', 'qs': 12, 'ps': 12, } """ qf: fields and the "boosts" `fieldOne^2.3 fieldTwo fieldThree^0.4` mm: (Minimum 'Should' Match) qs: Query Phrase Slop (in qf fields; affects matching).
if __name__ == "__main__": try: # try_verify_credentials() if IS_XRAY_ENABLE == "1": global_sdk_config.set_sdk_enabled(True) xray_recorder.configure( service='ecs', context_missing='LOG_ERROR', daemon_address='xray-service.kube-system:2000', plugins=('EC2Plugin', 'ECSPlugin')) libs_to_patch = ('boto3', 'requests') patch(libs_to_patch) else: global_sdk_config.set_sdk_enabled(False) event_loop() except ClientError as e: errlog.log("ClientError Agent Event Loop {} [{}] POD:{}".format( e.response['Error']['Code'], traceback.format_exc(), SELF_ID)) sys.exit(1) except Exception as e: errlog.log("Exception Agent Event Loop {} [{}] POD:{}".format( e, traceback.format_exc(), SELF_ID)) sys.exit(1)
def db(): patch(('sqlite3', )) return sqlite3.connect(":memory:")
import logging import telegram from telegram.ext import CommandHandler from sqlalchemy.dialects import postgresql as pg from aws_xray_sdk.core import xray_recorder, patch from ..schemas import User, Chat patch(['requests']) logger = logging.getLogger() def get_handlers(): return [CommandHandler('start', start)] @xray_recorder.capture("start_handler") def start(update: telegram.Update, context: telegram.ext.CallbackContext): bot = context.bot session = bot.db_session chat = update.effective_chat user = update.effective_user # Do nothing if chat already exists if session.query(Chat).get(chat.id): bot.send_message(chat_id=chat.id, text=f"You are already registered, {user.full_name}!") return
import psycopg2 import psycopg2.pool import pytest import testing.postgresql from aws_xray_sdk.core import patch from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.core.context import Context patch(('psycopg2', )) @pytest.fixture(autouse=True) def construct_ctx(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities() def test_execute_dsn_kwargs(): q = 'SELECT 1' with testing.postgresql.Postgresql() as postgresql: url = postgresql.url()
import boto3 import uuid import json import logging import os import datetime import decimal from botocore.exceptions import ClientError # パッチ適用 from aws_xray_sdk.core import patch patch(['boto3']) logger = logging.getLogger() logger.setLevel(logging.INFO) dynamodb = boto3.resource('dynamodb', region_name='ap-northeast-1') table = dynamodb.Table(os.getenv('TABLE_NAME')) def lambda_handler(event, context): try: userid = event['userid'] startdate = event['startdate'] try: response = table.get_item(Key={ 'userid': userid, 'startdate': startdate }) if 'Item' not in response: logging.info("Specified key is not found.")
import json import os import boto3 import codecs from botocore.exceptions import ClientError from aws_xray_sdk.core import xray_recorder, patch from aws_xray_sdk.core.context import Context from aws_xray_sdk.ext.flask.middleware import XRayMiddleware from flask import request, make_response from flask_lambda import FlaskLambda patch(('boto3',)) xray_recorder.configure( service='x-ray-profiling-experiment', dynamic_naming='*.execute-api.us-east-1.amazonaws.com/test*', context=Context() ) s3_client = boto3.resource('s3') app = FlaskLambda(__name__) XRayMiddleware(app, xray_recorder) HOST = os.environ.get('HOST') PORT = int(os.environ.get('PORT')) if 'PORT' in os.environ else None bucket = 'logs-test-861229788715' # health check endpoint @app.route('/health')
import sqlite3 import pytest from aws_xray_sdk.core import patch from aws_xray_sdk.core import xray_recorder patch(('sqlite3',)) db = sqlite3.connect(":memory:") @pytest.fixture(autouse=True) def construct_ctx(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities() def test_execute(): q = 'SELECT name FROM sqlite_master' db.execute(q) subsegment = xray_recorder.current_segment().subsegments[0] assert subsegment.name == ':memory:' sql = subsegment.sql
__author__ = 'Inigo Montoya ([email protected])' __vcs_id__ = '$Id$' __version__ = '$Version$' from flask import Flask, request from flask_restful import Resource, Api import os, json, logging import requests import boto3 #Setup AWS X-Ray so we get application metrics from aws_xray_sdk.core import xray_recorder, patch from aws_xray_sdk.ext.flask.middleware import XRayMiddleware xray_recorder.configure(context_missing='LOG_ERROR') xray_recorder.configure(service='CICDApiProxy') patch(('requests', )) #Secure Hash code - it works out our secret by hashing all the source files #A valid hash needs unchanged files in /unicorn_descriptions and secrethash.py, and the correct version numbers in app.py from secrethash import hasher # Set Unicorn Rentals backend API URL to proxy API requests too # We use AWS SSM Parameter Store, as is much easier and clearer than using ENVARS try: client = boto3.client('ssm') response = client.get_parameter(Name='BACKEND_API') BACKEND_API = response['Parameter']['Value'] except: print "SSM unavailable - do you have an IAM role" BACKEND_API = 'http://catsndogs.lol' print "Backend set to: {}".format(BACKEND_API)
import pytest import botocore.session from botocore import UNSIGNED from botocore.client import Config from botocore.exceptions import ClientError from pynamodb.attributes import UnicodeAttribute from pynamodb.models import Model from aws_xray_sdk.core import patch from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.core.context import Context patch(('pynamodb',)) @pytest.fixture(autouse=True) def construct_ctx(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities() def test_exception():
def patch_module(): patch(('pymysql', )) yield unpatch()
class XrayTracer(object): emitter = XrayEmitter() in_lambda = 'LAMBDA_TASK_ROOT' in os.environ use_daemon = 'AWS_XRAY_DAEMON_ADDRESS' in os.environ service_name = 'custodian' context = XrayContext() if HAVE_XRAY: xray_recorder.configure(emitter=use_daemon is False and emitter or None, context=context, sampling=True, context_missing='LOG_ERROR') patch(['boto3', 'requests']) logging.getLogger('aws_xray_sdk.core').setLevel(logging.ERROR) def __init__(self, ctx, config): self.ctx = ctx self.config = config or {} self.client = None self.metadata = {} @contextlib.contextmanager def subsegment(self, name): segment = xray_recorder.begin_subsegment(name) try: yield segment except Exception as e: stack = traceback.extract_stack(limit=xray_recorder.max_trace_back) segment.add_exception(e, stack) raise finally: xray_recorder.end_subsegment(time.time()) def __enter__(self): if self.client is None: self.client = self.ctx.session_factory(assume=False).client('xray') self.emitter.client = self.client if self.in_lambda: self.segment = xray_recorder.begin_subsegment(self.service_name) else: self.segment = xray_recorder.begin_segment(self.service_name, sampling=True) p = self.ctx.policy xray_recorder.put_annotation('policy', p.name) xray_recorder.put_annotation('resource', p.resource_type) if self.ctx.options.account_id: xray_recorder.put_annotation('account', self.ctx.options.account_id) def __exit__(self, exc_type=None, exc_value=None, exc_traceback=None): metadata = self.ctx.get_metadata(('api-stats', )) metadata.update(self.metadata) xray_recorder.put_metadata('custodian', metadata) if self.in_lambda: xray_recorder.end_subsegment() return xray_recorder.end_segment() if not self.use_daemon: self.emitter.flush() self.metadata.clear()
import logging import os from collections import Counter import boto3 from aws_xray_sdk.core import patch from boto3.dynamodb import types libraries = ("boto3",) patch(libraries) logger = logging.getLogger() logger.setLevel(os.environ["LOG_LEVEL"]) dynamodb = boto3.resource("dynamodb") table = dynamodb.Table(os.environ["TABLE_NAME"]) deserializer = types.TypeDeserializer() def dynamodb_to_python(dynamodb_item): """ Convert from dynamodb low level format to python dict """ return {k: deserializer.deserialize(v) for k, v in dynamodb_item.items()} def lambda_handler(event, context): """ Handle streams from DynamoDB table """
def create_app(script_info=None): # instantiate the app app = Flask(__name__) flask_bcrypt = Bcrypt(app) jwt = JWTManager(app) app.json_encoder = JSONEncoder # enable CORS CORS(app, resources={r'/*': {'origins': '*'}}) # TODO 9: Review X-ray setting patch_modules = ( 'boto3', 'botocore', 'pynamodb', 'requests', ) plugins = ('EC2Plugin', ) xray_recorder.configure(service='CloudAlbum', plugins=plugins, context_missing='LOG_ERROR', sampling=False) XRayMiddleware(app, xray_recorder) patch(patch_modules) # set config app_settings = os.getenv('APP_SETTINGS') app.config.from_object(app_settings) # set logger to STDOUT app.logger.addHandler(logging.StreamHandler(sys.stdout)) app.logger.setLevel(logging.DEBUG) # register blueprints from cloudalbum.api.users import users_blueprint app.register_blueprint(users_blueprint, url_prefix='/users') from cloudalbum.api.photos import photos_blueprint app.register_blueprint(photos_blueprint, url_prefix='/photos') from cloudalbum.api.admin import admin_blueprint app.register_blueprint(admin_blueprint, url_prefix='/admin') @jwt.token_in_blacklist_loader def check_if_token_in_blacklist_DB(decrypted_token): from cloudalbum.util.jwt_helper import is_blacklisted_token_set try: return is_blacklisted_token_set(decrypted_token) except Exception as e: app.logger.error(e) return make_response( jsonify({'msg': 'session already expired'}, 409)) # shell context for flask cli @app.shell_context_processor def ctx(): return {'app': app} return app
from copy import deepcopy from aws_xray_sdk.core import patch from aws_xray_sdk.core import xray_recorder from raster_analysis.boto import lambda_client, invoke_lambda from raster_analysis.globals import LOGGER, RASTER_ANALYSIS_LAMBDA_NAME patch(["boto3"]) @xray_recorder.capture("Fanout") def handler(event, context): tiles = event.get("tiles", []) payload_base = event["payload"] for tile in tiles: payload = deepcopy(payload_base) payload["tile"] = tile try: invoke_lambda(payload, RASTER_ANALYSIS_LAMBDA_NAME, lambda_client()) except Exception as e: LOGGER.error( f"Invoke raster analysis lambda failed for aws request id: {context.aws_request_id}, tile: {tile}" ) raise e
from functools import wraps import typing from aws_xray_sdk.core.context import MISSING_SEGMENT_MSG from aws_xray_sdk.core.exceptions.exceptions import SegmentNotFoundException from aws_xray_sdk.core import xray_recorder, patch from aws_xray_sdk.core.models.subsegment import Subsegment as xray_Subsegment import logging from logging import Logger logger = logging.getLogger(__name__) DSS_XRAY_TRACE = int(os.environ.get('DSS_XRAY_TRACE', '0')) > 0 # noqa patched = False if DSS_XRAY_TRACE and not patched: # noqa patch(('boto3', 'requests')) xray_recorder.configure(context_missing='LOG_ERROR') patched = True class XrayLoggerFilter(logging.Filter): def filter(self, record): if record.msg == MISSING_SEGMENT_MSG: return False try: entity = xray_recorder.get_trace_entity() except RecursionError: return True else: record.xray_trace_id = entity.trace_id if entity else "" record.msg = f"{record.xray_trace_id}\t{record.msg}"
'CRITICAL': logging.CRITICAL, 'ERROR': logging.ERROR, 'WARNING': logging.WARNING, 'INFO': logging.INFO, 'DEBUG': logging.DEBUG } if os.environ.get('LOG_LEVEL') in logging_map: logger.setLevel(logging_map[os.environ.get('LOG_LEVEL')]) else: logger.setLevel(logging.WARNING) if os.environ.get('ENABLE_XRAY') == 'true': try: from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.core import patch patch(['boto3', 'requests']) except ImportError: logger.warning( 'Unable to load aws_xray_sdk module. Xray functionality ' 'may be limited') events_to_ignore = ['5152'] def lambda_handler(event, context): host = os.environ['ES_ENDPOINT'] region = os.environ['REGION'] access_key = boto3.client('kms').decrypt( CiphertextBlob=b64decode(os.environ['ACCESS_KEY']))['Plaintext'] secret_key = boto3.client('kms').decrypt( CiphertextBlob=b64decode(os.environ['SECRET_KEY']))['Plaintext']
import pytest import aiobotocore from botocore.stub import Stubber, ANY from botocore.exceptions import ClientError from aws_xray_sdk.core import patch from aws_xray_sdk.core.async_context import AsyncContext from aws_xray_sdk.core import xray_recorder patch(('aiobotocore', )) @pytest.fixture(scope='function') def recorder(loop): """ Clean up before and after each test run """ xray_recorder.configure(service='test', sampling=False, context=AsyncContext(loop=loop)) xray_recorder.clear_trace_entities() yield xray_recorder xray_recorder.clear_trace_entities() async def test_describe_table(loop, recorder): segment = recorder.begin_segment('name') req_id = '1234' response = {