def create_app(script_info=None):
# instantiate the app
app = Flask(__name__)
flask_bcrypt = Bcrypt(app)
jwt = JWTManager(app)
app.json_encoder = JSONEncoder
# enable CORS
CORS(app, resources={r'/*': {'origins': '*'}})
# TODO 9: Review X-ray setting
patch_modules = (
'boto3',
'botocore',
'pynamodb',
'requests',
)
plugins = ('EC2Plugin', )
xray_recorder.configure(service='CloudAlbum',
plugins=plugins,
context_missing='LOG_ERROR',
sampling=False)
xray_recorder.begin_segment('cloudalbum')
XRayMiddleware(app, xray_recorder)
patch(patch_modules)
# set config
app_settings = os.getenv('APP_SETTINGS')
app.config.from_object(app_settings)
# set logger to STDOUT
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.DEBUG)
# Create database table, if it is not exists
with app.app_context():
create_table()
# register blueprints
from cloudalbum.api.users import users_blueprint
app.register_blueprint(users_blueprint, url_prefix='/users')
from cloudalbum.api.photos import photos_blueprint
app.register_blueprint(photos_blueprint, url_prefix='/photos')
from cloudalbum.api.admin import admin_blueprint
app.register_blueprint(admin_blueprint, url_prefix='/admin')
# shell context for flask cli
@app.shell_context_processor
def ctx():
return {'app': app}
return app
def construct_ctx():
"""
Clean up context storage on each test run and begin a segment
so that later subsegment can be attached. After each test run
it cleans up context storage again.
"""
xray_recorder.clear_trace_entities()
xray_recorder.begin_segment('name')
yield
xray_recorder.clear_trace_entities()
def connection(engine):
conn = engine.connect()
xray_recorder.configure(service='test', sampling=False, context=Context())
xray_recorder.clear_trace_entities()
xray_recorder.begin_segment('SQLAlchemyTest')
Session = XRaySessionMaker(bind=conn)
Base.metadata.create_all(engine)
session = Session()
yield session
xray_recorder.end_segment()
xray_recorder.clear_trace_entities()
async def test_ok_name(loop, recorder):
xray_recorder.begin_segment('name')
trace_config = aws_xray_trace_config(name='test')
status_code = 200
url = 'http://{}/status/{}?foo=bar'.format(BASE_URL, status_code)
async with ClientSession(loop=loop, trace_configs=[trace_config]) as session:
async with session.get(url):
pass
subsegment = xray_recorder.current_segment().subsegments[0]
assert subsegment.name == 'test'
def main():
while True:
# Start a segment
xray_recorder.begin_segment('processor')
process_messages()
# Close the segment
xray_recorder.end_segment('processor')
time.sleep(5)
def construct_ctx():
"""
Clean up context storage on each test run and begin a segment
so that later subsegment can be attached. After each test run
it cleans up context storage again.
"""
xray_recorder.configure(service='test', sampling=False, context=Context())
xray_recorder.clear_trace_entities()
xray_recorder.begin_segment('name')
yield
xray_recorder.clear_trace_entities()
def put_item():
trace_id = request.headers.get(X_RAY_HEADER_TRACE)
parent_id = request.headers.get(X_RAY_HEADER_PARENT)
xray_recorder.begin_segment(name='API3', parent_id=parent_id ,traceid=trace_id, sampling=1)
response = jsonify({'api3': "ok"} )
response.status_code = 200
response.mimetype = "application/json"
xray_recorder.end_segment()
return response
def session(request):
"""Test Fixture to Create DataBase Tables and start a trace segment"""
xray_recorder.configure(service='test',
sampling=False,
context=Context(),
stream_sql=request.param)
xray_recorder.clear_trace_entities()
xray_recorder.begin_segment('SQLAlchemyTest')
db.create_all()
yield
xray_recorder.end_segment()
xray_recorder.clear_trace_entities()
def session():
"""Test Fixture to Create DataBase Tables and start a trace segment"""
engine = create_engine('sqlite:///:memory:')
xray_recorder.configure(service='test', sampling=False, context=Context())
xray_recorder.clear_trace_entities()
xray_recorder.begin_segment('SQLAlchemyTest')
Session = XRaySessionMaker(bind=engine)
Base.metadata.create_all(engine)
session = Session()
yield session
xray_recorder.end_segment()
xray_recorder.clear_trace_entities()
def func_setup(request, user_class):
xray_recorder.stream_sql = request.param
xray_recorder.clear_trace_entities()
xray_recorder.begin_segment('name')
try:
user_class.create_table()
yield
finally:
xray_recorder.clear_trace_entities()
try:
user_class.delete_table()
finally:
xray_recorder.end_segment()
def test_function(self):
xray_recorder.begin_segment("test_function")
file = open("event.json", "rb")
try:
event = file.read()
logger.warning("## EVENT")
context = {"requestid": "1234"}
result = handler(event, context)
print(str(result))
self.assertRegex(json.loads(result)["id"], "cs_test_.*", "Should match")
finally:
file.close()
file.close()
xray_recorder.end_segment()
def test_function(self):
xray_recorder.begin_segment('test_function')
file = open('event.json', 'rb')
try:
ba = bytearray(file.read())
event = jsonpickle.decode(ba)
logger.warning('## EVENT')
logger.warning(jsonpickle.encode(event))
context = {'requestid': '1234'}
result = handler(event, context)
print(str(result))
finally:
file.close()
file.close()
xray_recorder.end_segment()
def test_function(self):
xray_recorder.begin_segment("test_function")
file = open("in.json", "rb")
try:
ba = bytearray(file.read())
event = jsonpickle.decode(ba)
logger.warning("## EVENT")
logger.warning(jsonpickle.encode(event))
context = {"requestid": "1234"}
result = handler(event, context)
print(str(result))
self.assertRegex(str(result), "FunctionCount", "Should match")
finally:
file.close()
file.close()
xray_recorder.end_segment()
def __call__(self, request):
sampling_decision = None
meta = request.META
xray_header = construct_xray_header(meta)
# a segment name is required
name = calculate_segment_name(meta.get(HOST_KEY), xray_recorder)
sampling_req = {
'host': meta.get(HOST_KEY),
'method': request.method,
'path': request.path,
'service': name,
}
sampling_decision = calculate_sampling_decision(
trace_header=xray_header,
recorder=xray_recorder,
sampling_req=sampling_req,
)
if self.in_lambda_ctx:
segment = xray_recorder.begin_subsegment(name)
else:
segment = xray_recorder.begin_segment(
name=name,
traceid=xray_header.root,
parent_id=xray_header.parent,
sampling=sampling_decision,
)
segment.save_origin_trace_header(xray_header)
segment.put_http_meta(http.URL, request.build_absolute_uri())
segment.put_http_meta(http.METHOD, request.method)
if meta.get(USER_AGENT_KEY):
segment.put_http_meta(http.USER_AGENT, meta.get(USER_AGENT_KEY))
if meta.get(X_FORWARDED_KEY):
# X_FORWARDED_FOR may come from untrusted source so we
# need to set the flag to true as additional information
segment.put_http_meta(http.CLIENT_IP, meta.get(X_FORWARDED_KEY))
segment.put_http_meta(http.X_FORWARDED_FOR, True)
elif meta.get(REMOTE_ADDR_KEY):
segment.put_http_meta(http.CLIENT_IP, meta.get(REMOTE_ADDR_KEY))
response = self.get_response(request)
segment.put_http_meta(http.STATUS, response.status_code)
if response.has_header(CONTENT_LENGTH_KEY):
length = int(response[CONTENT_LENGTH_KEY])
segment.put_http_meta(http.CONTENT_LENGTH, length)
response[http.XRAY_HEADER] = prepare_response_header(
xray_header, segment)
if self.in_lambda_ctx:
xray_recorder.end_subsegment()
else:
xray_recorder.end_segment()
return response
async def test_fault(loop, recorder):
xray_recorder.begin_segment('name')
trace_config = aws_xray_trace_config()
status_code = 500
url = 'http://{}/status/{}'.format(BASE_URL, status_code)
async with ClientSession(loop=loop, trace_configs=[trace_config]) as session:
async with session.put(url):
pass
subsegment = xray_recorder.current_segment().subsegments[0]
assert subsegment.name == url
assert subsegment.fault
http_meta = subsegment.http
assert http_meta['request']['url'] == url
assert http_meta['request']['method'] == 'PUT'
assert http_meta['response']['status'] == status_code
async def test_invalid_url(loop, recorder):
xray_recorder.begin_segment('name')
trace_config = aws_xray_trace_config()
async with ClientSession(loop=loop, trace_configs=[trace_config]) as session:
try:
async with session.get('http://doesnt.exist'):
pass
except Exception:
# prevent uncatch exception from breaking test run
pass
subsegment = xray_recorder.current_segment().subsegments[0]
assert subsegment.namespace == LOCAL_NAMESPACE
assert subsegment.fault
exception = subsegment.cause['exceptions'][0]
assert exception.type == 'ClientConnectorError'
def ddb_message():
xray_recorder.begin_segment('ddb_message')
message_id = os.environ.get('MESSAGE_ID')
app.logger.info(message_id)
try:
response = table.get_item(Key={'id': int(message_id)})
item = response['Item']
app.logger.info("----------")
app.logger.info(item)
app.logger.info("----------")
app.logger.info(json.dumps(item, indent=4, cls=DecimalEncoder))
except Exception:
app.logger.info(traceback.print_exc())
xray_recorder.end_segment()
return item['message']
def construct_ctx():
"""
Clean up context storage on each test run and begin a segment
so that later subsegment can be attached. After each test run
it cleans up context storage again.
"""
from aws_xray_sdk.ext.httplib import unpatch, reset_ignored
patch(('httplib',))
xray_recorder.configure(service='test', sampling=False, context=Context())
xray_recorder.clear_trace_entities()
xray_recorder.begin_segment('name')
yield
xray_recorder.clear_trace_entities()
unpatch()
reset_ignored()
async def test_ok(loop, recorder):
xray_recorder.begin_segment('name')
trace_config = aws_xray_trace_config()
status_code = 200
url = 'http://{}/status/{}?foo=bar'.format(BASE_URL, status_code)
async with ClientSession(loop=loop, trace_configs=[trace_config]) as session:
async with session.get(url):
pass
subsegment = xray_recorder.current_segment().subsegments[0]
assert subsegment.name == strip_url(url)
assert subsegment.namespace == REMOTE_NAMESPACE
http_meta = subsegment.http
assert http_meta['request']['url'] == url
assert http_meta['request']['method'] == 'GET'
assert http_meta['response']['status'] == status_code
def engine():
"""
Clean up context storage on each test run and begin a segment
so that later subsegment can be attached. After each test run
it cleans up context storage again.
"""
from aws_xray_sdk.ext.sqlalchemy_core import unpatch
patch(('sqlalchemy_core',))
engine = create_engine('sqlite:///:memory:')
xray_recorder.configure(service='test', sampling=False, context=Context())
xray_recorder.begin_segment('name')
Base.metadata.create_all(engine)
xray_recorder.clear_trace_entities()
xray_recorder.begin_segment('name')
yield engine
xray_recorder.clear_trace_entities()
unpatch()
def __init__(self):
parser.add_argument('x-traceid', location='headers')
parser.add_argument('x-parentid', location='headers')
rgs = parser.parse_args()
self.traceid = rgs["x-traceid"]
self.parentid = rgs["x-parentid"]
segment = xray_recorder.begin_segment('api_%s_todos'%(API_VERSION), traceid = self.traceid, parent_id=self.parentid,sampling=1)
segment.put_http_meta(http.URL, 'api-%s.flask.sample'%(API_VERSION))
logger.info("get todos from api ver%s"%(API_VERSION))
async def test_throttle(loop, recorder):
xray_recorder.begin_segment('name')
trace_config = aws_xray_trace_config()
status_code = 429
url = 'http://{}/status/{}'.format(BASE_URL, status_code)
async with ClientSession(loop=loop,
trace_configs=[trace_config]) as session:
async with session.head(url):
pass
subsegment = xray_recorder.current_segment().subsegments[0]
assert subsegment.name == get_hostname(url)
assert subsegment.error
assert subsegment.throttle
http_meta = subsegment.http
assert http_meta['request']['url'] == strip_url(url)
assert http_meta['request']['method'] == 'HEAD'
assert http_meta['response']['status'] == status_code
def main():
xray_recorder.begin_segment('main_function')
file = open('event.json', 'rb')
try:
# read sample event
ba = bytearray(file.read())
event = jsonpickle.decode(ba)
logger.warning('## EVENT')
logger.warning(jsonpickle.encode(event))
# create sample context
context = {'requestid': '1234'}
# invoke handler
result = upload_task_issues(event, context)
# print response
print('## RESPONSE')
print(str(result))
finally:
file.close()
file.close()
xray_recorder.end_segment()
def invoke_lambda(batches, m_id):
xray_recorder.begin_segment('Invoke mapper Lambda')
'''
lambda invoke function
'''
batch = [k.key for k in batches[m_id - 1]]
xray_recorder.current_segment().put_annotation(
"batch_for_mapper_" + str(m_id), str(batch))
resp = lambda_client.invoke(FunctionName=mapper_lambda_name,
InvocationType='RequestResponse',
Payload=json.dumps({
"bucket": bucket,
"keys": batch,
"jobBucket": job_bucket,
"jobId": job_id,
"mapperId": m_id
}))
out = eval(resp['Payload'].read())
mapper_outputs.append(out)
print "mapper output", out
xray_recorder.end_segment()
def s3_upload_file():
segment = xray_recorder.begin_segment('1')
subsegment = xray_recorder.begin_subsegment('requesting form data/upload')
bucket = request.form.get('bucket')
filename = request.form.get('filename')
s3_upload_svc = boto3.resource('s3')
bucket_selected = s3_upload_svc.Bucket(bucket)
with open(filename, 'rb') as f:
bucket_selected.upload_fileobj(f, Key=filename)
#confirm if tag argument is needed.
xray_recorder.end_subsegment()
xray_recorder.end_segment()
return "OK"
def hello_world():
# Start a segment
segment = xray_recorder.begin_segment('hello_world')
# Start a subsegment
subsegment = xray_recorder.begin_subsegment('get-s3-data-files')
version = "version-05"
cluster_name = 'not_set'
resp = {}
data = {}
try:
logging.info("metadata_uri:" +
os.environ['ECS_CONTAINER_METADATA_URI'])
resp = requests.get(url=os.environ['ECS_CONTAINER_METADATA_URI'] +
'/task')
data = resp.json()
except Exception as e:
logging.error("error getting metadata:" + str(e))
if 'Cluster' in data:
cluster_arn = data['Cluster']
cluster_list = cluster_arn.split('/')
cluster_name = cluster_list[1]
else:
cluster_name = 'not_set'
logging.info("cluster:" + cluster_name)
client = boto3.client('s3')
response = client.list_buckets()
logging.info("s3 response: %s", response['Owner']['DisplayName'])
# Close the subsegment and segment
xray_recorder.end_subsegment()
try:
get_file()
except Exception as e:
logging.error("get_File error")
xray_recorder.end_segment()
return json.dumps({"version": version}), 500
def dynamodb():
xray_recorder.begin_segment(name='API1', sampling=1)
current_segment = xray_recorder.current_segment()
headers = {
X_RAY_HEADER_TRACE: current_segment.trace_id,
X_RAY_HEADER_PARENT: current_segment.id
}
url = "http://" + os.environ['API2_HOST'] + ":5000"
r = requests.get(url, headers=headers)
data = r.json()
response = jsonify({
'api1': 'ok',
'api2': data['api2'],
'api3': data['api3']
})
response.status_code = 200
xray_recorder.end_segment()
return response
def test_id_generation_default_sampling_true():
segment = xray_recorder.begin_segment('segment_name', sampling=True)
# Start and end a subsegment
subsegment = xray_recorder.begin_subsegment('subsegment_name')
xray_recorder.end_subsegment()
# Close the segment
xray_recorder.end_segment()
assert segment.id != '0000000000000000'
assert segment.trace_id != '1-00000000-000000000000000000000000'
assert subsegment.id != '0000000000000000'
assert subsegment.trace_id != '1-00000000-000000000000000000000000'
assert subsegment.parent_id != '0000000000000000'
def test_id_generation_noop_false():
os.environ['AWS_XRAY_NOOP_ID'] = 'FALSE'
segment = xray_recorder.begin_segment('segment_name', sampling=False)
# Start and end a subsegment
subsegment = xray_recorder.begin_subsegment('subsegment_name')
xray_recorder.end_subsegment()
# Close the segment
xray_recorder.end_segment()
assert segment.id != '0000000000000000'
assert segment.trace_id != '1-00000000-000000000000000000000000'
assert subsegment.id != '0000000000000000'
assert subsegment.trace_id != '1-00000000-000000000000000000000000'
assert subsegment.parent_id != '0000000000000000'
def __enter__(self):
if self.client is None:
self.client = self.ctx.session_factory(assume=False).client('xray')
self.emitter.client = self.client
if self.in_lambda:
self.segment = xray_recorder.begin_subsegment(self.service_name)
else:
self.segment = xray_recorder.begin_segment(
self.service_name, sampling=True)
p = self.ctx.policy
xray_recorder.put_annotation('policy', p.name)
xray_recorder.put_annotation('resource', p.resource_type)
if self.ctx.options.account_id:
xray_recorder.put_annotation('account', self.ctx.options.account_id)
def __enter__(self):
if self.client is None:
self.client = self.ctx.session_factory(assume=False).client('xray')
self.emitter.client = self.client
if self.in_lambda:
self.segment = xray_recorder.begin_subsegment(self.service_name)
else:
self.segment = xray_recorder.begin_segment(
self.service_name, sampling=True)
p = self.ctx.policy
xray_recorder.put_annotation('policy', p.name)
xray_recorder.put_annotation('resource', p.resource_type)
if self.ctx.options.account_id:
xray_recorder.put_annotation('account', self.ctx.options.account_id)
