def get_guids(ts):
guids = []
# get the guid values up to x days ago
queryres = ddb.query(IndexName='timest',
KeyConditionExpression=Key('visible').eq('y')
& Key('timest').gt(str(ts)))
for x in queryres['Items']:
if 'guid' in x:
if x['guid'] not in guids:
guids.append(x['guid'])
# paginate the query in case more than 100 results are returned
while 'LastEvaluatedKey' in queryres:
queryres = ddb.query(ExclusiveStartKey=queryres['LastEvaluatedKey'],
IndexName='timest',
KeyConditionExpression=Key('visible').eq('y')
& Key('timest').gt(str(ts)))
for x in queryres['Items']:
if 'guid' in x:
if x['guid'] not in guids:
guids.append(x['guid'])
xray_recorder.current_subsegment().put_annotation('postcountguid',
str(len(guids)))
print('guids found: ' + str(len(guids)))
return guids
def get_file():
"""
demonstrate intentional error
:return:
"""
result = None
try:
# add request context information to the trace for this subsegment
document = xray_recorder.current_subsegment()
document.put_annotation("filename", "hello.txt")
request = {'requestId': '123-123', 'value1': 20}
document.put_metadata('key', request, 'namespace')
# intentional file not found error to demonstrate x-ray
s3 = boto3.resource('s3')
result = s3.Object('dummybucket',
'hello.txt').download_file('/tmp/hello.txt')
except Exception as e:
logging.error("download failed: %s", str(e))
finally:
if result is None:
sub_segment = xray_recorder.current_subsegment()
sub_segment.apply_status_code(500)
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
'hello.txt')
def query_by_partition_key(self, partition_key, partition_value):
response = self.db_table.query(
KeyConditionExpression=Key(partition_key).eq(partition_value))
status_code = response['ResponseMetadata']['HTTPStatusCode']
xray_recorder.current_subsegment().put_annotation(
'get_response', status_code)
return response.get('Items')
def process_event(e: dict):
repository_name = e['repository_name']
xray_recorder.current_subsegment().put_metadata('repository_name',
repository_name)
association = associations[repository_name]
# Find all branches...
branches = []
response = cc.list_branches(repositoryName=repository_name)
branches.extend(response['branches'])
while ("NextToken" in response):
response = cc.list_branches(repositoryName=repository_name,
nextToken=response['NextToken'])
branches.extend(response['branches'])
# Process each branch...
xray_recorder.current_subsegment().put_metadata('branches', str(branches))
for branch in branches:
timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f")
response = cg.create_code_review(
Name='Analyze-{}'.format(timestamp),
RepositoryAssociationArn=association['AssociationArn'],
Type={
"RepositoryAnalysis": {
"RepositoryHead": {
"BranchName": branch
}
}
})
print(dumps({"repository_name": repository_name, "branches": branches}))
def get_guids():
a = []
c = s.query(IndexName='allts',
KeyConditionExpression=Key('allts').eq('y'),
ProjectionExpression='guid')
for x in c['Items']:
if 'guid' in x:
if x['guid'] not in a:
a.append(x['guid'])
while 'LastEvaluatedKey' in c:
c = s.query(ExclusiveStartKey=c['LastEvaluatedKey'],
IndexName='allts',
KeyConditionExpression=Key('allts').eq('y'),
ProjectionExpression='guid')
for x in c['Items']:
if 'guid' in x:
if x['guid'] not in a:
a.append(x['guid'])
xray_recorder.current_subsegment().put_annotation('postcountguid',
str(len(a)))
print('len allts ' + str(len(a)))
return a
def put_dynamo(s, timest, title, desc, link, source, auth, guid, tags,
category):
if len(desc) == 0:
desc = '...'
xray_recorder.current_subsegment().put_annotation('ddbposturl', str(link))
xray_recorder.current_subsegment().put_annotation(
'ddbpostfields',
str(
str(timest) + ' ' + title + ' ' + desc + ' ' + link + ' ' +
source + ' ' + auth + ' ' + guid + ' ' + tags + ' ' + category))
s.put_item(TableName=os.environ['dynamo_table'],
Item={
'timest': timest,
'title': title,
'desc': desc,
'link': link,
'source': source,
'author': auth,
'tag': tags,
'lower-tag': tags.lower(),
'allts': 'y',
'guid': guid,
'tags': tags,
'category': category
})
def send_message_to_proxy(msg):
print("[ {} ] Sending message to proxy. Message contents: {}".format(
datetime.datetime.utcnow(), msg))
proxy_comm = yield connect_to_address("tcp://" + proxy_endpoint + ":8989",
connection_args={})
if proxy_comm.closed():
print(
"[ {} ] [ERROR] Connection attempt #1 to proxy failed... Trying again..."
.format(datetime.datetime.utcnow()))
max_tries = 4
num_tries = 2
while proxy_comm.closed() and num_tries <= max_tries:
proxy_comm = yield connect_to_address("tcp://" + proxy_endpoint +
":8989",
connection_args={})
print(
"[ {} ] [ERROR] Connection attempt #{} to proxy failed... Trying again..."
.format(datetime.datetime.utcnow(), num_tries))
num_tries += 1
if proxy_comm.closed():
print("[ {} ] [ERROR] Cannot connect to proxy...")
return False
bytes_written = yield proxy_comm.write(msg, serializers=["msgpack"])
xray_recorder.current_subsegment().put_annotation("bytes_written_to_proxy",
str(bytes_written))
print("[ {} ] Wrote {} bytes to the Redis Proxy.".format(
datetime.datetime.utcnow(), bytes_written))
return True
def get_secret():
secret_name = "starAlliances/dev/seatMap/provider"
region_name = "us-east-1"
try:
xray_recorder.begin_subsegment('init_client')
# Create a Secrets Manager client
session = boto3.session.Session()
client = session.client(
service_name='secretsmanager',
region_name=region_name
)
xray_recorder.current_subsegment().put_annotation('init_client', 'done')
finally:
xray_recorder.end_subsegment()
# In this sample we only handle the specific exceptions for the 'GetSecretValue' API.
# See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
# We rethrow the exception by default.
try:
get_secret_value_response = client.get_secret_value(
SecretId=secret_name
)
except ClientError as e:
if e.response['Error']['Code'] == 'DecryptionFailureException':
# Secrets Manager can't decrypt the protected secret text using the provided KMS key.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'InternalServiceErrorException':
# An error occurred on the server side.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'InvalidParameterException':
# You provided an invalid value for a parameter.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'InvalidRequestException':
# You provided a parameter value that is not valid for the current state of the resource.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'ResourceNotFoundException':
# We can't find the resource that you asked for.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
else:
# Decrypts secret using the associated KMS CMK.
# Depending on whether the secret is a string or binary, one of these fields will be populated.
if 'SecretString' in get_secret_value_response:
secret = get_secret_value_response['SecretString']
return secret
else:
decoded_binary_secret = base64.b64decode(get_secret_value_response['SecretBinary'])
return decoded_binary_secret
# Your code goes here.
return ""
def download_file(url, project_htm):
xray_recorder.current_subsegment().put_metadata('project_htm', project_htm)
headers = {'Accept-Encoding': 'identity'}
print(url)
r = requests.get(url, headers=headers)
with open(project_htm, 'w+') as f:
f.write(r.text)
def handler(event, context):
xray_recorder.begin_subsegment('handler')
xray_recorder.current_subsegment().put_annotation('Lambda',
context.function_name)
res = fetchall_athena(event["query"], event["type"])
logger.info(res)
# AddToDdbStat(res)
return {'result': True}
def download_apk(project_htm, apk_href):
xray_recorder.current_subsegment().put_metadata('apk_href', apk_href)
outdir = path.dirname(project_htm)
outfile = path.join(outdir, path.basename(apk_href))
if path.exists(outfile):
return
r = requests.get(apk_href)
with open(outfile, 'wb') as f:
f.write(r.content)
def handler(event, context):
# get requestor http headers
seg = xray_recorder.begin_subsegment('dynamo-session')
ip = str(event['headers']['X-Forwarded-For']).split(',')[0]
ua = str(event['headers']['User-Agent'])
print('IP: ' + str(event['headers']['X-Forwarded-For']))
# print request headers in cloudwatch for debug purposes
xray_recorder.end_subsegment()
# clean the given url path and print debug
xray_recorder.begin_subsegment('path-find')
pa = event['path']
xray_recorder.current_subsegment().put_annotation('clientip', ip)
xray_recorder.current_subsegment().put_annotation('useragent', ua)
xray_recorder.current_subsegment().put_annotation('urlpath', pa)
# check whether a tag, category, url or no path argument was given
npa, tag = check_path(pa)
#parse_debug(event)
xray_recorder.end_subsegment()
xray_recorder.begin_subsegment('html-parse')
# if a url was submitted, redirect to it with a 301
if npa == 'redir':
print('301 ***', str(event['headers']['User-Agent']),
str(event['headers']['Host']), pa, '', tag)
xray_recorder.current_subsegment().put_annotation('statuscode', '301')
return {'statusCode': '301', 'headers': {'Location': tag}}
# else parse the html page
else:
url = check_ua(ua)
html = parse_html(d, npa, tag, url)
print('200 ***', str(event['headers']['User-Agent']),
str(event['headers']['Host']), pa, url, tag)
xray_recorder.current_subsegment().put_annotation('statuscode', '200')
return {
'statusCode': '200',
'body': str(html),
'headers': {
'Content-Type': 'text/html',
'charset': 'utf-8'
}
}
xray_recorder.end_subsegment()
def update_dynamo_event_counter(self,
event_name,
event_datetime,
event_count=1):
response = self.db_table.update_item(
Key={
'EventId': event_name,
'EventDay': event_datetime
},
ExpressionAttributeValues={":eventCount": event_count})
status_code = response['ResponseMetadata']['HTTPStatusCode']
xray_recorder.current_subsegment().put_annotation(
'get_response', status_code)
def download_file():
with open(f'/tmp/{BUCKET_KEY}', 'wb') as data:
try:
S3_CLIENT.download_fileobj(BUCKET_NAME, BUCKET_KEY, data)
status_code = 200
print(f"/tmp contents: {os.listdir('/tmp')}"
) # Check if the file was created
except Exception as exp:
print(exp)
status_code = 500
finally:
xray_recorder.current_subsegment().put_annotation(
'get_response', status_code)
def parse_html(d, npa, tag, url):
po, co = get_posts(d, npa, tag, url)
xray_recorder.current_subsegment().put_annotation('post-count', co)
h = '<html><head><meta charset="UTF-8"><title>marek\'s serverless demo</title>'
h += load_file('main.css') + '</head>'
h += '<body><center>' + counter(url, co)
h += '<table width = 800px><tr><td><h1><center id = "top">marek\'s serverless AWS blog</center></h1></td></tr>'
h += '<tr><td id = "urls">' + generate_urls(d, npa,
url) + '</td></tr></table><br>'
h += '<table>'
h += po
h += '</table>' + load_file('collapse.js') + ' ' + c + '</body></html>'
return h
def handler(event, context):
try:
xray_recorder.begin_subsegment('handler')
xray_recorder.current_subsegment().put_annotation(
'Lambda', context.function_name)
#logger.info(event)
if 'body' not in event:
logger.error('Missing parameters')
return {'result': False, 'msg': 'Missing parameters'}
body = json.loads(event['body'])
logger.info(body)
s3.Object(s3_bucket, 'selfie-reports/' + body['id'] + '.json').delete()
logger.info('DELETED: selfie-reports/' + body['id'] + '.json')
# Get the lastest csv for deletion
get_last_modified = lambda obj: int(obj['LastModified'].strftime('%s'))
objs = client.list_objects_v2(Bucket=s3_bucket,
Prefix='selfie-ath-results/' +
body['emotion'])['Contents']
file_list = [
obj['Key']
for obj in sorted(objs, key=get_last_modified, reverse=True)
]
for csv_file in file_list:
if "metadata" in csv_file:
continue
s3.Object(s3_bucket, csv_file).delete()
logger.info('DELETED: ' + csv_file)
break
payload["type"] = "delselfie"
response = client.invoke(FunctionName=AthDispatcherLambdaName,
InvocationType='Event',
Payload=json.dumps(payload))
logger.info(response)
return {
'result': True,
'selfie': 'selfie-reports/' + body['id'] + '.json',
'csv': 'selfie-ath-results/' + body['emotion'] + '/' + csv_file
}
except Exception as e:
logger.error('Something went wrong: ' + str(e))
return {'result': False, 'msg': str(e)}
def _get_wiki_url(_url, q):
BASE_URL = _url
payload = {}
resp = {"statusCode": 400}
HOT_TOPICS = [
'cholas', 'cheras', 'pandyas', 'pallavas', 'sangam_era', 'kural'
]
if q:
q = q.split('/')[-1]
if not q:
q = random.choice(HOT_TOPICS)
try:
random_sleep()
if _trigger_exception():
xray_recorder.put_annotation("SIMULATED_ERRORS", "True")
xray_recorder.begin_subsegment("BRITTLE_LEGACY_APP")
d = xray_recorder.current_subsegment()
d.put_annotation("MANUALLY_TRIGGERRED_IN_SUBSEGMENT", "True")
xray_recorder.end_subsegment()
raise Exception("RANDOM_ERROR: Simulate Mystique Failure")
r1 = requests.get(f'{BASE_URL}/{q}', params=payload)
xray_recorder.put_metadata('RESPONSE', resp)
resp["statusCode"] = r1.status_code
z = r1.json()
resp["body"] = json.dumps({"message": z["body"]["message"]})
_ddb_put_item(resp)
except Exception as e:
resp["body"] = json.dumps({"message": str(e)})
return resp
async def end_subsegment(session, trace_config_ctx, params):
if trace_config_ctx.give_up:
return
subsegment = xray_recorder.current_subsegment()
subsegment.put_http_meta(http.STATUS, params.response.status)
xray_recorder.end_subsegment()
def get_handler(event, _):
"""
Handler for GetItem
"""
try:
name = event["pathParameters"]["name"]
except KeyError:
return error(400, "missing 'name' parameter")
if not name:
return error(400, "missing 'name' parameter")
# When using sam local, current_subsegment() returns None
subsegment = xray_recorder.current_subsegment()
if subsegment:
subsegment.put_annotation("name", name)
try:
item = get_item(name)
return {
"statusCode": 200,
"body": json.dumps(item, default=dec_encode)
}
except ValueError as exc:
return error(404, str(exc))
except Exception as exc:
return error(500, str(exc))
def lambda_handler(event, context):
"""
AWS Lambda handler
This method is invoked by the API Gateway: /Prod/first/{proxy+} endpoint.
"""
xray_subsegment = xray_recorder.current_subsegment()
xray_subsegment.put_annotation(
'application', '{{ cookiecutter.project_name.lower().replace('
',' - ') }}')
xray_subsegment.put_metadata(
'event', event, '{{ cookiecutter.project_name.lower().replace('
',' - ') }}')
try:
subsegment = xray_recorder.begin_subsegment('message')
message = {
'Id': uuid.uuid4().hex,
'Count': random.random() * 100,
}
subsegment.put_metadata(
'message', message, '{{ cookiecutter.project_name.lower().replace('
',' - ') }}')
xray_recorder.end_subsegment()
return {"statusCode": 200, "body": json.dumps(message)}
except Exception as err: # pragma: no cover
logger.error(str(err))
traceback.print_exc()
raise err
def put_handler(event, _):
"""
Handler for PutItem
"""
try:
name = event["pathParameters"]["name"]
except KeyError:
return error(400, "missing 'name' parameter")
if not name:
return error(400, "missing 'name' parameter")
# When using sam local, current_subsegment() returns None
subsegment = xray_recorder.current_subsegment()
if subsegment:
subsegment.put_annotation("name", name)
item = json.loads(event["body"])
try:
put_item(name, item)
return {"statusCode": 200, "body": json.dumps({"message": "Success"})}
except Exception as exc:
return error(500, str(exc))
def failed(self, event):
subsegment = xray_recorder.current_subsegment()
subsegment.add_fault_flag()
subsegment.put_annotation('mongodb_duration_micros',
event.duration_micros)
subsegment.put_metadata('failure', event.failure)
xray_recorder.end_subsegment()
def succeeded(self, event):
subsegment = xray_recorder.current_subsegment()
subsegment.put_annotation('mongodb_duration_micros',
event.duration_micros)
if self.record_full_documents:
subsegment.put_metadata('mongodb_reply', event.reply)
xray_recorder.end_subsegment()
def _inject_header(wrapped, instance, args, kwargs):
request = args[0]
headers = getattr(request, 'headers', {})
inject_trace_header(headers, xray_recorder.current_subsegment())
setattr(request, 'headers', headers)
return wrapped(*args, **kwargs)
def decompose_args(method, url, body, headers, encode_chunked=False):
# skip httplib tracing for SDK built-in centralized sampling pollers
if (('/GetSamplingRules' in args or '/SamplingTargets' in args)
and type(instance).__name__
== 'botocore.awsrequest.AWSHTTPConnection'):
return wrapped(*args, **kwargs)
# Only injects headers when the subsegment for the outgoing
# calls are opened successfully.
subsegment = None
try:
subsegment = xray_recorder.current_subsegment()
except SegmentNotFoundException:
pass
if subsegment:
inject_trace_header(headers, subsegment)
ssl_cxt = getattr(instance, '_context', None)
scheme = 'https' if ssl_cxt and type(
ssl_cxt).__name__ == 'SSLContext' else 'http'
xray_url = '{}://{}{}'.format(scheme, instance.host, url)
xray_data = _XRay_Data(method, instance.host, xray_url)
setattr(instance, _XRAY_PROP, xray_data)
# we add a segment here in case connect fails
return xray_recorder.record_subsegment(
wrapped,
instance,
args,
kwargs,
name=strip_url(xray_data.url),
namespace='remote',
meta_processor=http_send_request_processor)
def put_object_into_s3(bucket_name, bucket_key, body):
subsegment = xray_recorder.current_subsegment()
response = s3_client.put_object(Bucket=bucket_name,
Key=bucket_key,
Body=body)
status_code = response['ResponseMetadata']['HTTPStatusCode']
subsegment.put_annotation('put_response', status_code)
def download_intermediate_results(filename):
"""Download a file from S3
Parameters
----------
filename: string, required
Name of the file in S3 source bucket (OpenAQ)
Returns
-------
processed_file: string
Local path to downloaded file
"""
xray_recorder.begin_subsegment('## download_data_file')
subsegment = xray_recorder.current_subsegment()
subsegment.put_metadata('filename', f's3://{output_bucket}/{filename}')
try:
processed_file = os.path.join('/tmp', os.path.basename(filename))
s3.download_file(output_bucket, filename, processed_file)
subsegment.put_annotation('DATA_DOWNLOAD', 'SUCCESS')
except botocore.exceptions.ClientError as e:
subsegment.put_annotation('DATA_DOWNLOAD', 'FAILURE')
log.error(f'Unable to download rsult file: {filename}')
log.debug(e)
raise
xray_recorder.end_subsegment()
return processed_file
def extract_dd_trace_context(event):
"""
Extract Datadog trace context from the Lambda `event` object.
Write the context to a global `dd_trace_context`, so the trace
can be continued on the outgoing requests with the context injected.
Save the context to an X-Ray subsegment's metadata field, so the X-Ray
trace can be converted to a Datadog trace in the Datadog backend with
the correct context.
"""
global dd_trace_context
headers = event.get('headers', {})
trace_id = headers.get(TraceHeader.TRACE_ID)
parent_id = headers.get(TraceHeader.PARENT_ID)
sampling_priority = headers.get(TraceHeader.SAMPLING_PRIORITY)
if trace_id and parent_id and sampling_priority:
dd_trace_context = {
'trace-id': trace_id,
'parent-id': parent_id,
'sampling-priority': sampling_priority,
}
xray_recorder.begin_subsegment(XraySubsegment.NAME)
subsegment = xray_recorder.current_subsegment()
subsegment.put_metadata(
XraySubsegment.KEY,
dd_trace_context,
XraySubsegment.NAMESPACE
)
xray_recorder.end_subsegment()
else:
# AWS Lambda runtime caches global variables between invocations,
# reset to avoid using the context from the last invocation.
dd_trace_context = {}
def upload_final_results(results):
"""Upload a file to S3
Parameters
----------
results: string, required
Name of the local file with final results
"""
xray_recorder.begin_subsegment('## upload_final_results')
subsegment = xray_recorder.current_subsegment()
subsegment.put_metadata(
'filename',
f's3://{output_bucket}/lambda-etl-refarch/output/{results}')
results_path = os.path.join('/tmp', results)
# upload to target S3 bucket
try:
response = s3.upload_file(
results_path, output_bucket,
'lambda-etl-refarch/output/{}'.format(results))
log.info(
f"Uploaded final results to s3://{output_bucket}/lambda-etl-refarch/output/{results}"
)
subsegment.put_annotation('FINAL_RESULTS_UPLOAD', 'SUCCESS')
except botocore.exceptions.ClientError as e:
subsegment.put_annotation('FINAL_RESULTS_UPLOAD', 'FAILURE')
log.error(f'Unable to upload final results: {results}')
log.debug(e)
raise
xray_recorder.end_subsegment()
def xray_render(self, context):
template_name = self.name or getattr(context, 'template_name', None)
if template_name:
name = str(template_name)
subsegment = xray_recorder.current_subsegment()
subsegment.name = name
return Template._xray_original_render(self, context)
def get_message():
{% if cookiecutter.include_xray == "y" -%}
"""
You can create a sub-segment specifically to a function
then capture what sub-segment that is inside your code
and you can add annotations that will be indexed by X-Ray
for example: put_annotation("operation", "query_db")
"""
# Only run xray in the AWS Lambda environment
if runs_on_aws_lambda():
xray_subsegment = xray_recorder.current_subsegment()
xray_subsegment.put_annotation("key", "value")
{% endif -%}
return { "hello": "world" }
