def create_app(script_info=None): # instantiate the app app = Flask(__name__) flask_bcrypt = Bcrypt(app) jwt = JWTManager(app) app.json_encoder = JSONEncoder # enable CORS CORS(app, resources={r'/*': {'origins': '*'}}) # TODO 9: Review X-ray setting patch_modules = ( 'boto3', 'botocore', 'pynamodb', 'requests', ) plugins = ('EC2Plugin', ) xray_recorder.configure(service='CloudAlbum', plugins=plugins, context_missing='LOG_ERROR', sampling=False) xray_recorder.begin_segment('cloudalbum') XRayMiddleware(app, xray_recorder) patch(patch_modules) # set config app_settings = os.getenv('APP_SETTINGS') app.config.from_object(app_settings) # set logger to STDOUT app.logger.addHandler(logging.StreamHandler(sys.stdout)) app.logger.setLevel(logging.DEBUG) # Create database table, if it is not exists with app.app_context(): create_table() # register blueprints from cloudalbum.api.users import users_blueprint app.register_blueprint(users_blueprint, url_prefix='/users') from cloudalbum.api.photos import photos_blueprint app.register_blueprint(photos_blueprint, url_prefix='/photos') from cloudalbum.api.admin import admin_blueprint app.register_blueprint(admin_blueprint, url_prefix='/admin') # shell context for flask cli @app.shell_context_processor def ctx(): return {'app': app} return app
def construct_ctx(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities()
def connection(engine): conn = engine.connect() xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('SQLAlchemyTest') Session = XRaySessionMaker(bind=conn) Base.metadata.create_all(engine) session = Session() yield session xray_recorder.end_segment() xray_recorder.clear_trace_entities()
async def test_ok_name(loop, recorder): xray_recorder.begin_segment('name') trace_config = aws_xray_trace_config(name='test') status_code = 200 url = 'http://{}/status/{}?foo=bar'.format(BASE_URL, status_code) async with ClientSession(loop=loop, trace_configs=[trace_config]) as session: async with session.get(url): pass subsegment = xray_recorder.current_segment().subsegments[0] assert subsegment.name == 'test'
def main(): while True: # Start a segment xray_recorder.begin_segment('processor') process_messages() # Close the segment xray_recorder.end_segment('processor') time.sleep(5)
def construct_ctx(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities()
def put_item(): trace_id = request.headers.get(X_RAY_HEADER_TRACE) parent_id = request.headers.get(X_RAY_HEADER_PARENT) xray_recorder.begin_segment(name='API3', parent_id=parent_id ,traceid=trace_id, sampling=1) response = jsonify({'api3': "ok"} ) response.status_code = 200 response.mimetype = "application/json" xray_recorder.end_segment() return response
def session(request): """Test Fixture to Create DataBase Tables and start a trace segment""" xray_recorder.configure(service='test', sampling=False, context=Context(), stream_sql=request.param) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('SQLAlchemyTest') db.create_all() yield xray_recorder.end_segment() xray_recorder.clear_trace_entities()
def session(): """Test Fixture to Create DataBase Tables and start a trace segment""" engine = create_engine('sqlite:///:memory:') xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('SQLAlchemyTest') Session = XRaySessionMaker(bind=engine) Base.metadata.create_all(engine) session = Session() yield session xray_recorder.end_segment() xray_recorder.clear_trace_entities()
def func_setup(request, user_class): xray_recorder.stream_sql = request.param xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') try: user_class.create_table() yield finally: xray_recorder.clear_trace_entities() try: user_class.delete_table() finally: xray_recorder.end_segment()
def test_function(self): xray_recorder.begin_segment("test_function") file = open("event.json", "rb") try: event = file.read() logger.warning("## EVENT") context = {"requestid": "1234"} result = handler(event, context) print(str(result)) self.assertRegex(json.loads(result)["id"], "cs_test_.*", "Should match") finally: file.close() file.close() xray_recorder.end_segment()
def test_function(self): xray_recorder.begin_segment('test_function') file = open('event.json', 'rb') try: ba = bytearray(file.read()) event = jsonpickle.decode(ba) logger.warning('## EVENT') logger.warning(jsonpickle.encode(event)) context = {'requestid': '1234'} result = handler(event, context) print(str(result)) finally: file.close() file.close() xray_recorder.end_segment()
def test_function(self): xray_recorder.begin_segment("test_function") file = open("in.json", "rb") try: ba = bytearray(file.read()) event = jsonpickle.decode(ba) logger.warning("## EVENT") logger.warning(jsonpickle.encode(event)) context = {"requestid": "1234"} result = handler(event, context) print(str(result)) self.assertRegex(str(result), "FunctionCount", "Should match") finally: file.close() file.close() xray_recorder.end_segment()
def __call__(self, request): sampling_decision = None meta = request.META xray_header = construct_xray_header(meta) # a segment name is required name = calculate_segment_name(meta.get(HOST_KEY), xray_recorder) sampling_req = { 'host': meta.get(HOST_KEY), 'method': request.method, 'path': request.path, 'service': name, } sampling_decision = calculate_sampling_decision( trace_header=xray_header, recorder=xray_recorder, sampling_req=sampling_req, ) if self.in_lambda_ctx: segment = xray_recorder.begin_subsegment(name) else: segment = xray_recorder.begin_segment( name=name, traceid=xray_header.root, parent_id=xray_header.parent, sampling=sampling_decision, ) segment.save_origin_trace_header(xray_header) segment.put_http_meta(http.URL, request.build_absolute_uri()) segment.put_http_meta(http.METHOD, request.method) if meta.get(USER_AGENT_KEY): segment.put_http_meta(http.USER_AGENT, meta.get(USER_AGENT_KEY)) if meta.get(X_FORWARDED_KEY): # X_FORWARDED_FOR may come from untrusted source so we # need to set the flag to true as additional information segment.put_http_meta(http.CLIENT_IP, meta.get(X_FORWARDED_KEY)) segment.put_http_meta(http.X_FORWARDED_FOR, True) elif meta.get(REMOTE_ADDR_KEY): segment.put_http_meta(http.CLIENT_IP, meta.get(REMOTE_ADDR_KEY)) response = self.get_response(request) segment.put_http_meta(http.STATUS, response.status_code) if response.has_header(CONTENT_LENGTH_KEY): length = int(response[CONTENT_LENGTH_KEY]) segment.put_http_meta(http.CONTENT_LENGTH, length) response[http.XRAY_HEADER] = prepare_response_header( xray_header, segment) if self.in_lambda_ctx: xray_recorder.end_subsegment() else: xray_recorder.end_segment() return response
async def test_fault(loop, recorder): xray_recorder.begin_segment('name') trace_config = aws_xray_trace_config() status_code = 500 url = 'http://{}/status/{}'.format(BASE_URL, status_code) async with ClientSession(loop=loop, trace_configs=[trace_config]) as session: async with session.put(url): pass subsegment = xray_recorder.current_segment().subsegments[0] assert subsegment.name == url assert subsegment.fault http_meta = subsegment.http assert http_meta['request']['url'] == url assert http_meta['request']['method'] == 'PUT' assert http_meta['response']['status'] == status_code
async def test_invalid_url(loop, recorder): xray_recorder.begin_segment('name') trace_config = aws_xray_trace_config() async with ClientSession(loop=loop, trace_configs=[trace_config]) as session: try: async with session.get('http://doesnt.exist'): pass except Exception: # prevent uncatch exception from breaking test run pass subsegment = xray_recorder.current_segment().subsegments[0] assert subsegment.namespace == LOCAL_NAMESPACE assert subsegment.fault exception = subsegment.cause['exceptions'][0] assert exception.type == 'ClientConnectorError'
def ddb_message(): xray_recorder.begin_segment('ddb_message') message_id = os.environ.get('MESSAGE_ID') app.logger.info(message_id) try: response = table.get_item(Key={'id': int(message_id)}) item = response['Item'] app.logger.info("----------") app.logger.info(item) app.logger.info("----------") app.logger.info(json.dumps(item, indent=4, cls=DecimalEncoder)) except Exception: app.logger.info(traceback.print_exc()) xray_recorder.end_segment() return item['message']
def construct_ctx(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ from aws_xray_sdk.ext.httplib import unpatch, reset_ignored patch(('httplib',)) xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield xray_recorder.clear_trace_entities() unpatch() reset_ignored()
async def test_ok(loop, recorder): xray_recorder.begin_segment('name') trace_config = aws_xray_trace_config() status_code = 200 url = 'http://{}/status/{}?foo=bar'.format(BASE_URL, status_code) async with ClientSession(loop=loop, trace_configs=[trace_config]) as session: async with session.get(url): pass subsegment = xray_recorder.current_segment().subsegments[0] assert subsegment.name == strip_url(url) assert subsegment.namespace == REMOTE_NAMESPACE http_meta = subsegment.http assert http_meta['request']['url'] == url assert http_meta['request']['method'] == 'GET' assert http_meta['response']['status'] == status_code
def engine(): """ Clean up context storage on each test run and begin a segment so that later subsegment can be attached. After each test run it cleans up context storage again. """ from aws_xray_sdk.ext.sqlalchemy_core import unpatch patch(('sqlalchemy_core',)) engine = create_engine('sqlite:///:memory:') xray_recorder.configure(service='test', sampling=False, context=Context()) xray_recorder.begin_segment('name') Base.metadata.create_all(engine) xray_recorder.clear_trace_entities() xray_recorder.begin_segment('name') yield engine xray_recorder.clear_trace_entities() unpatch()
def __init__(self): parser.add_argument('x-traceid', location='headers') parser.add_argument('x-parentid', location='headers') rgs = parser.parse_args() self.traceid = rgs["x-traceid"] self.parentid = rgs["x-parentid"] segment = xray_recorder.begin_segment('api_%s_todos'%(API_VERSION), traceid = self.traceid, parent_id=self.parentid,sampling=1) segment.put_http_meta(http.URL, 'api-%s.flask.sample'%(API_VERSION)) logger.info("get todos from api ver%s"%(API_VERSION))
async def test_throttle(loop, recorder): xray_recorder.begin_segment('name') trace_config = aws_xray_trace_config() status_code = 429 url = 'http://{}/status/{}'.format(BASE_URL, status_code) async with ClientSession(loop=loop, trace_configs=[trace_config]) as session: async with session.head(url): pass subsegment = xray_recorder.current_segment().subsegments[0] assert subsegment.name == get_hostname(url) assert subsegment.error assert subsegment.throttle http_meta = subsegment.http assert http_meta['request']['url'] == strip_url(url) assert http_meta['request']['method'] == 'HEAD' assert http_meta['response']['status'] == status_code
def main(): xray_recorder.begin_segment('main_function') file = open('event.json', 'rb') try: # read sample event ba = bytearray(file.read()) event = jsonpickle.decode(ba) logger.warning('## EVENT') logger.warning(jsonpickle.encode(event)) # create sample context context = {'requestid': '1234'} # invoke handler result = upload_task_issues(event, context) # print response print('## RESPONSE') print(str(result)) finally: file.close() file.close() xray_recorder.end_segment()
def invoke_lambda(batches, m_id): xray_recorder.begin_segment('Invoke mapper Lambda') ''' lambda invoke function ''' batch = [k.key for k in batches[m_id - 1]] xray_recorder.current_segment().put_annotation( "batch_for_mapper_" + str(m_id), str(batch)) resp = lambda_client.invoke(FunctionName=mapper_lambda_name, InvocationType='RequestResponse', Payload=json.dumps({ "bucket": bucket, "keys": batch, "jobBucket": job_bucket, "jobId": job_id, "mapperId": m_id })) out = eval(resp['Payload'].read()) mapper_outputs.append(out) print "mapper output", out xray_recorder.end_segment()
def s3_upload_file(): segment = xray_recorder.begin_segment('1') subsegment = xray_recorder.begin_subsegment('requesting form data/upload') bucket = request.form.get('bucket') filename = request.form.get('filename') s3_upload_svc = boto3.resource('s3') bucket_selected = s3_upload_svc.Bucket(bucket) with open(filename, 'rb') as f: bucket_selected.upload_fileobj(f, Key=filename) #confirm if tag argument is needed. xray_recorder.end_subsegment() xray_recorder.end_segment() return "OK"
def hello_world(): # Start a segment segment = xray_recorder.begin_segment('hello_world') # Start a subsegment subsegment = xray_recorder.begin_subsegment('get-s3-data-files') version = "version-05" cluster_name = 'not_set' resp = {} data = {} try: logging.info("metadata_uri:" + os.environ['ECS_CONTAINER_METADATA_URI']) resp = requests.get(url=os.environ['ECS_CONTAINER_METADATA_URI'] + '/task') data = resp.json() except Exception as e: logging.error("error getting metadata:" + str(e)) if 'Cluster' in data: cluster_arn = data['Cluster'] cluster_list = cluster_arn.split('/') cluster_name = cluster_list[1] else: cluster_name = 'not_set' logging.info("cluster:" + cluster_name) client = boto3.client('s3') response = client.list_buckets() logging.info("s3 response: %s", response['Owner']['DisplayName']) # Close the subsegment and segment xray_recorder.end_subsegment() try: get_file() except Exception as e: logging.error("get_File error") xray_recorder.end_segment() return json.dumps({"version": version}), 500
def dynamodb(): xray_recorder.begin_segment(name='API1', sampling=1) current_segment = xray_recorder.current_segment() headers = { X_RAY_HEADER_TRACE: current_segment.trace_id, X_RAY_HEADER_PARENT: current_segment.id } url = "http://" + os.environ['API2_HOST'] + ":5000" r = requests.get(url, headers=headers) data = r.json() response = jsonify({ 'api1': 'ok', 'api2': data['api2'], 'api3': data['api3'] }) response.status_code = 200 xray_recorder.end_segment() return response
def test_id_generation_default_sampling_true(): segment = xray_recorder.begin_segment('segment_name', sampling=True) # Start and end a subsegment subsegment = xray_recorder.begin_subsegment('subsegment_name') xray_recorder.end_subsegment() # Close the segment xray_recorder.end_segment() assert segment.id != '0000000000000000' assert segment.trace_id != '1-00000000-000000000000000000000000' assert subsegment.id != '0000000000000000' assert subsegment.trace_id != '1-00000000-000000000000000000000000' assert subsegment.parent_id != '0000000000000000'
def test_id_generation_noop_false(): os.environ['AWS_XRAY_NOOP_ID'] = 'FALSE' segment = xray_recorder.begin_segment('segment_name', sampling=False) # Start and end a subsegment subsegment = xray_recorder.begin_subsegment('subsegment_name') xray_recorder.end_subsegment() # Close the segment xray_recorder.end_segment() assert segment.id != '0000000000000000' assert segment.trace_id != '1-00000000-000000000000000000000000' assert subsegment.id != '0000000000000000' assert subsegment.trace_id != '1-00000000-000000000000000000000000' assert subsegment.parent_id != '0000000000000000'
def __enter__(self): if self.client is None: self.client = self.ctx.session_factory(assume=False).client('xray') self.emitter.client = self.client if self.in_lambda: self.segment = xray_recorder.begin_subsegment(self.service_name) else: self.segment = xray_recorder.begin_segment( self.service_name, sampling=True) p = self.ctx.policy xray_recorder.put_annotation('policy', p.name) xray_recorder.put_annotation('resource', p.resource_type) if self.ctx.options.account_id: xray_recorder.put_annotation('account', self.ctx.options.account_id)
def __enter__(self): if self.client is None: self.client = self.ctx.session_factory(assume=False).client('xray') self.emitter.client = self.client if self.in_lambda: self.segment = xray_recorder.begin_subsegment(self.service_name) else: self.segment = xray_recorder.begin_segment( self.service_name, sampling=True) p = self.ctx.policy xray_recorder.put_annotation('policy', p.name) xray_recorder.put_annotation('resource', p.resource_type) if self.ctx.options.account_id: xray_recorder.put_annotation('account', self.ctx.options.account_id)