def get_guids(ts): guids = [] # get the guid values up to x days ago queryres = ddb.query(IndexName='timest', KeyConditionExpression=Key('visible').eq('y') & Key('timest').gt(str(ts))) for x in queryres['Items']: if 'guid' in x: if x['guid'] not in guids: guids.append(x['guid']) # paginate the query in case more than 100 results are returned while 'LastEvaluatedKey' in queryres: queryres = ddb.query(ExclusiveStartKey=queryres['LastEvaluatedKey'], IndexName='timest', KeyConditionExpression=Key('visible').eq('y') & Key('timest').gt(str(ts))) for x in queryres['Items']: if 'guid' in x: if x['guid'] not in guids: guids.append(x['guid']) xray_recorder.current_subsegment().put_annotation('postcountguid', str(len(guids))) print('guids found: ' + str(len(guids))) return guids
def get_file(): """ demonstrate intentional error :return: """ result = None try: # add request context information to the trace for this subsegment document = xray_recorder.current_subsegment() document.put_annotation("filename", "hello.txt") request = {'requestId': '123-123', 'value1': 20} document.put_metadata('key', request, 'namespace') # intentional file not found error to demonstrate x-ray s3 = boto3.resource('s3') result = s3.Object('dummybucket', 'hello.txt').download_file('/tmp/hello.txt') except Exception as e: logging.error("download failed: %s", str(e)) finally: if result is None: sub_segment = xray_recorder.current_subsegment() sub_segment.apply_status_code(500) raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), 'hello.txt')
def query_by_partition_key(self, partition_key, partition_value): response = self.db_table.query( KeyConditionExpression=Key(partition_key).eq(partition_value)) status_code = response['ResponseMetadata']['HTTPStatusCode'] xray_recorder.current_subsegment().put_annotation( 'get_response', status_code) return response.get('Items')
def process_event(e: dict): repository_name = e['repository_name'] xray_recorder.current_subsegment().put_metadata('repository_name', repository_name) association = associations[repository_name] # Find all branches... branches = [] response = cc.list_branches(repositoryName=repository_name) branches.extend(response['branches']) while ("NextToken" in response): response = cc.list_branches(repositoryName=repository_name, nextToken=response['NextToken']) branches.extend(response['branches']) # Process each branch... xray_recorder.current_subsegment().put_metadata('branches', str(branches)) for branch in branches: timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f") response = cg.create_code_review( Name='Analyze-{}'.format(timestamp), RepositoryAssociationArn=association['AssociationArn'], Type={ "RepositoryAnalysis": { "RepositoryHead": { "BranchName": branch } } }) print(dumps({"repository_name": repository_name, "branches": branches}))
def get_guids(): a = [] c = s.query(IndexName='allts', KeyConditionExpression=Key('allts').eq('y'), ProjectionExpression='guid') for x in c['Items']: if 'guid' in x: if x['guid'] not in a: a.append(x['guid']) while 'LastEvaluatedKey' in c: c = s.query(ExclusiveStartKey=c['LastEvaluatedKey'], IndexName='allts', KeyConditionExpression=Key('allts').eq('y'), ProjectionExpression='guid') for x in c['Items']: if 'guid' in x: if x['guid'] not in a: a.append(x['guid']) xray_recorder.current_subsegment().put_annotation('postcountguid', str(len(a))) print('len allts ' + str(len(a))) return a
def put_dynamo(s, timest, title, desc, link, source, auth, guid, tags, category): if len(desc) == 0: desc = '...' xray_recorder.current_subsegment().put_annotation('ddbposturl', str(link)) xray_recorder.current_subsegment().put_annotation( 'ddbpostfields', str( str(timest) + ' ' + title + ' ' + desc + ' ' + link + ' ' + source + ' ' + auth + ' ' + guid + ' ' + tags + ' ' + category)) s.put_item(TableName=os.environ['dynamo_table'], Item={ 'timest': timest, 'title': title, 'desc': desc, 'link': link, 'source': source, 'author': auth, 'tag': tags, 'lower-tag': tags.lower(), 'allts': 'y', 'guid': guid, 'tags': tags, 'category': category })
def send_message_to_proxy(msg): print("[ {} ] Sending message to proxy. Message contents: {}".format( datetime.datetime.utcnow(), msg)) proxy_comm = yield connect_to_address("tcp://" + proxy_endpoint + ":8989", connection_args={}) if proxy_comm.closed(): print( "[ {} ] [ERROR] Connection attempt #1 to proxy failed... Trying again..." .format(datetime.datetime.utcnow())) max_tries = 4 num_tries = 2 while proxy_comm.closed() and num_tries <= max_tries: proxy_comm = yield connect_to_address("tcp://" + proxy_endpoint + ":8989", connection_args={}) print( "[ {} ] [ERROR] Connection attempt #{} to proxy failed... Trying again..." .format(datetime.datetime.utcnow(), num_tries)) num_tries += 1 if proxy_comm.closed(): print("[ {} ] [ERROR] Cannot connect to proxy...") return False bytes_written = yield proxy_comm.write(msg, serializers=["msgpack"]) xray_recorder.current_subsegment().put_annotation("bytes_written_to_proxy", str(bytes_written)) print("[ {} ] Wrote {} bytes to the Redis Proxy.".format( datetime.datetime.utcnow(), bytes_written)) return True
def get_secret(): secret_name = "starAlliances/dev/seatMap/provider" region_name = "us-east-1" try: xray_recorder.begin_subsegment('init_client') # Create a Secrets Manager client session = boto3.session.Session() client = session.client( service_name='secretsmanager', region_name=region_name ) xray_recorder.current_subsegment().put_annotation('init_client', 'done') finally: xray_recorder.end_subsegment() # In this sample we only handle the specific exceptions for the 'GetSecretValue' API. # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html # We rethrow the exception by default. try: get_secret_value_response = client.get_secret_value( SecretId=secret_name ) except ClientError as e: if e.response['Error']['Code'] == 'DecryptionFailureException': # Secrets Manager can't decrypt the protected secret text using the provided KMS key. # Deal with the exception here, and/or rethrow at your discretion. raise e elif e.response['Error']['Code'] == 'InternalServiceErrorException': # An error occurred on the server side. # Deal with the exception here, and/or rethrow at your discretion. raise e elif e.response['Error']['Code'] == 'InvalidParameterException': # You provided an invalid value for a parameter. # Deal with the exception here, and/or rethrow at your discretion. raise e elif e.response['Error']['Code'] == 'InvalidRequestException': # You provided a parameter value that is not valid for the current state of the resource. # Deal with the exception here, and/or rethrow at your discretion. raise e elif e.response['Error']['Code'] == 'ResourceNotFoundException': # We can't find the resource that you asked for. # Deal with the exception here, and/or rethrow at your discretion. raise e else: # Decrypts secret using the associated KMS CMK. # Depending on whether the secret is a string or binary, one of these fields will be populated. if 'SecretString' in get_secret_value_response: secret = get_secret_value_response['SecretString'] return secret else: decoded_binary_secret = base64.b64decode(get_secret_value_response['SecretBinary']) return decoded_binary_secret # Your code goes here. return ""
def download_file(url, project_htm): xray_recorder.current_subsegment().put_metadata('project_htm', project_htm) headers = {'Accept-Encoding': 'identity'} print(url) r = requests.get(url, headers=headers) with open(project_htm, 'w+') as f: f.write(r.text)
def handler(event, context): xray_recorder.begin_subsegment('handler') xray_recorder.current_subsegment().put_annotation('Lambda', context.function_name) res = fetchall_athena(event["query"], event["type"]) logger.info(res) # AddToDdbStat(res) return {'result': True}
def download_apk(project_htm, apk_href): xray_recorder.current_subsegment().put_metadata('apk_href', apk_href) outdir = path.dirname(project_htm) outfile = path.join(outdir, path.basename(apk_href)) if path.exists(outfile): return r = requests.get(apk_href) with open(outfile, 'wb') as f: f.write(r.content)
def handler(event, context): # get requestor http headers seg = xray_recorder.begin_subsegment('dynamo-session') ip = str(event['headers']['X-Forwarded-For']).split(',')[0] ua = str(event['headers']['User-Agent']) print('IP: ' + str(event['headers']['X-Forwarded-For'])) # print request headers in cloudwatch for debug purposes xray_recorder.end_subsegment() # clean the given url path and print debug xray_recorder.begin_subsegment('path-find') pa = event['path'] xray_recorder.current_subsegment().put_annotation('clientip', ip) xray_recorder.current_subsegment().put_annotation('useragent', ua) xray_recorder.current_subsegment().put_annotation('urlpath', pa) # check whether a tag, category, url or no path argument was given npa, tag = check_path(pa) #parse_debug(event) xray_recorder.end_subsegment() xray_recorder.begin_subsegment('html-parse') # if a url was submitted, redirect to it with a 301 if npa == 'redir': print('301 ***', str(event['headers']['User-Agent']), str(event['headers']['Host']), pa, '', tag) xray_recorder.current_subsegment().put_annotation('statuscode', '301') return {'statusCode': '301', 'headers': {'Location': tag}} # else parse the html page else: url = check_ua(ua) html = parse_html(d, npa, tag, url) print('200 ***', str(event['headers']['User-Agent']), str(event['headers']['Host']), pa, url, tag) xray_recorder.current_subsegment().put_annotation('statuscode', '200') return { 'statusCode': '200', 'body': str(html), 'headers': { 'Content-Type': 'text/html', 'charset': 'utf-8' } } xray_recorder.end_subsegment()
def update_dynamo_event_counter(self, event_name, event_datetime, event_count=1): response = self.db_table.update_item( Key={ 'EventId': event_name, 'EventDay': event_datetime }, ExpressionAttributeValues={":eventCount": event_count}) status_code = response['ResponseMetadata']['HTTPStatusCode'] xray_recorder.current_subsegment().put_annotation( 'get_response', status_code)
def download_file(): with open(f'/tmp/{BUCKET_KEY}', 'wb') as data: try: S3_CLIENT.download_fileobj(BUCKET_NAME, BUCKET_KEY, data) status_code = 200 print(f"/tmp contents: {os.listdir('/tmp')}" ) # Check if the file was created except Exception as exp: print(exp) status_code = 500 finally: xray_recorder.current_subsegment().put_annotation( 'get_response', status_code)
def parse_html(d, npa, tag, url): po, co = get_posts(d, npa, tag, url) xray_recorder.current_subsegment().put_annotation('post-count', co) h = '<html><head><meta charset="UTF-8"><title>marek\'s serverless demo</title>' h += load_file('main.css') + '</head>' h += '<body><center>' + counter(url, co) h += '<table width = 800px><tr><td><h1><center id = "top">marek\'s serverless AWS blog</center></h1></td></tr>' h += '<tr><td id = "urls">' + generate_urls(d, npa, url) + '</td></tr></table><br>' h += '<table>' h += po h += '</table>' + load_file('collapse.js') + ' ' + c + '</body></html>' return h
def handler(event, context): try: xray_recorder.begin_subsegment('handler') xray_recorder.current_subsegment().put_annotation( 'Lambda', context.function_name) #logger.info(event) if 'body' not in event: logger.error('Missing parameters') return {'result': False, 'msg': 'Missing parameters'} body = json.loads(event['body']) logger.info(body) s3.Object(s3_bucket, 'selfie-reports/' + body['id'] + '.json').delete() logger.info('DELETED: selfie-reports/' + body['id'] + '.json') # Get the lastest csv for deletion get_last_modified = lambda obj: int(obj['LastModified'].strftime('%s')) objs = client.list_objects_v2(Bucket=s3_bucket, Prefix='selfie-ath-results/' + body['emotion'])['Contents'] file_list = [ obj['Key'] for obj in sorted(objs, key=get_last_modified, reverse=True) ] for csv_file in file_list: if "metadata" in csv_file: continue s3.Object(s3_bucket, csv_file).delete() logger.info('DELETED: ' + csv_file) break payload["type"] = "delselfie" response = client.invoke(FunctionName=AthDispatcherLambdaName, InvocationType='Event', Payload=json.dumps(payload)) logger.info(response) return { 'result': True, 'selfie': 'selfie-reports/' + body['id'] + '.json', 'csv': 'selfie-ath-results/' + body['emotion'] + '/' + csv_file } except Exception as e: logger.error('Something went wrong: ' + str(e)) return {'result': False, 'msg': str(e)}
def _get_wiki_url(_url, q): BASE_URL = _url payload = {} resp = {"statusCode": 400} HOT_TOPICS = [ 'cholas', 'cheras', 'pandyas', 'pallavas', 'sangam_era', 'kural' ] if q: q = q.split('/')[-1] if not q: q = random.choice(HOT_TOPICS) try: random_sleep() if _trigger_exception(): xray_recorder.put_annotation("SIMULATED_ERRORS", "True") xray_recorder.begin_subsegment("BRITTLE_LEGACY_APP") d = xray_recorder.current_subsegment() d.put_annotation("MANUALLY_TRIGGERRED_IN_SUBSEGMENT", "True") xray_recorder.end_subsegment() raise Exception("RANDOM_ERROR: Simulate Mystique Failure") r1 = requests.get(f'{BASE_URL}/{q}', params=payload) xray_recorder.put_metadata('RESPONSE', resp) resp["statusCode"] = r1.status_code z = r1.json() resp["body"] = json.dumps({"message": z["body"]["message"]}) _ddb_put_item(resp) except Exception as e: resp["body"] = json.dumps({"message": str(e)}) return resp
async def end_subsegment(session, trace_config_ctx, params): if trace_config_ctx.give_up: return subsegment = xray_recorder.current_subsegment() subsegment.put_http_meta(http.STATUS, params.response.status) xray_recorder.end_subsegment()
def get_handler(event, _): """ Handler for GetItem """ try: name = event["pathParameters"]["name"] except KeyError: return error(400, "missing 'name' parameter") if not name: return error(400, "missing 'name' parameter") # When using sam local, current_subsegment() returns None subsegment = xray_recorder.current_subsegment() if subsegment: subsegment.put_annotation("name", name) try: item = get_item(name) return { "statusCode": 200, "body": json.dumps(item, default=dec_encode) } except ValueError as exc: return error(404, str(exc)) except Exception as exc: return error(500, str(exc))
def lambda_handler(event, context): """ AWS Lambda handler This method is invoked by the API Gateway: /Prod/first/{proxy+} endpoint. """ xray_subsegment = xray_recorder.current_subsegment() xray_subsegment.put_annotation( 'application', '{{ cookiecutter.project_name.lower().replace(' ',' - ') }}') xray_subsegment.put_metadata( 'event', event, '{{ cookiecutter.project_name.lower().replace(' ',' - ') }}') try: subsegment = xray_recorder.begin_subsegment('message') message = { 'Id': uuid.uuid4().hex, 'Count': random.random() * 100, } subsegment.put_metadata( 'message', message, '{{ cookiecutter.project_name.lower().replace(' ',' - ') }}') xray_recorder.end_subsegment() return {"statusCode": 200, "body": json.dumps(message)} except Exception as err: # pragma: no cover logger.error(str(err)) traceback.print_exc() raise err
def put_handler(event, _): """ Handler for PutItem """ try: name = event["pathParameters"]["name"] except KeyError: return error(400, "missing 'name' parameter") if not name: return error(400, "missing 'name' parameter") # When using sam local, current_subsegment() returns None subsegment = xray_recorder.current_subsegment() if subsegment: subsegment.put_annotation("name", name) item = json.loads(event["body"]) try: put_item(name, item) return {"statusCode": 200, "body": json.dumps({"message": "Success"})} except Exception as exc: return error(500, str(exc))
def failed(self, event): subsegment = xray_recorder.current_subsegment() subsegment.add_fault_flag() subsegment.put_annotation('mongodb_duration_micros', event.duration_micros) subsegment.put_metadata('failure', event.failure) xray_recorder.end_subsegment()
def succeeded(self, event): subsegment = xray_recorder.current_subsegment() subsegment.put_annotation('mongodb_duration_micros', event.duration_micros) if self.record_full_documents: subsegment.put_metadata('mongodb_reply', event.reply) xray_recorder.end_subsegment()
def _inject_header(wrapped, instance, args, kwargs): request = args[0] headers = getattr(request, 'headers', {}) inject_trace_header(headers, xray_recorder.current_subsegment()) setattr(request, 'headers', headers) return wrapped(*args, **kwargs)
def decompose_args(method, url, body, headers, encode_chunked=False): # skip httplib tracing for SDK built-in centralized sampling pollers if (('/GetSamplingRules' in args or '/SamplingTargets' in args) and type(instance).__name__ == 'botocore.awsrequest.AWSHTTPConnection'): return wrapped(*args, **kwargs) # Only injects headers when the subsegment for the outgoing # calls are opened successfully. subsegment = None try: subsegment = xray_recorder.current_subsegment() except SegmentNotFoundException: pass if subsegment: inject_trace_header(headers, subsegment) ssl_cxt = getattr(instance, '_context', None) scheme = 'https' if ssl_cxt and type( ssl_cxt).__name__ == 'SSLContext' else 'http' xray_url = '{}://{}{}'.format(scheme, instance.host, url) xray_data = _XRay_Data(method, instance.host, xray_url) setattr(instance, _XRAY_PROP, xray_data) # we add a segment here in case connect fails return xray_recorder.record_subsegment( wrapped, instance, args, kwargs, name=strip_url(xray_data.url), namespace='remote', meta_processor=http_send_request_processor)
def put_object_into_s3(bucket_name, bucket_key, body): subsegment = xray_recorder.current_subsegment() response = s3_client.put_object(Bucket=bucket_name, Key=bucket_key, Body=body) status_code = response['ResponseMetadata']['HTTPStatusCode'] subsegment.put_annotation('put_response', status_code)
def download_intermediate_results(filename): """Download a file from S3 Parameters ---------- filename: string, required Name of the file in S3 source bucket (OpenAQ) Returns ------- processed_file: string Local path to downloaded file """ xray_recorder.begin_subsegment('## download_data_file') subsegment = xray_recorder.current_subsegment() subsegment.put_metadata('filename', f's3://{output_bucket}/{filename}') try: processed_file = os.path.join('/tmp', os.path.basename(filename)) s3.download_file(output_bucket, filename, processed_file) subsegment.put_annotation('DATA_DOWNLOAD', 'SUCCESS') except botocore.exceptions.ClientError as e: subsegment.put_annotation('DATA_DOWNLOAD', 'FAILURE') log.error(f'Unable to download rsult file: {filename}') log.debug(e) raise xray_recorder.end_subsegment() return processed_file
def extract_dd_trace_context(event): """ Extract Datadog trace context from the Lambda `event` object. Write the context to a global `dd_trace_context`, so the trace can be continued on the outgoing requests with the context injected. Save the context to an X-Ray subsegment's metadata field, so the X-Ray trace can be converted to a Datadog trace in the Datadog backend with the correct context. """ global dd_trace_context headers = event.get('headers', {}) trace_id = headers.get(TraceHeader.TRACE_ID) parent_id = headers.get(TraceHeader.PARENT_ID) sampling_priority = headers.get(TraceHeader.SAMPLING_PRIORITY) if trace_id and parent_id and sampling_priority: dd_trace_context = { 'trace-id': trace_id, 'parent-id': parent_id, 'sampling-priority': sampling_priority, } xray_recorder.begin_subsegment(XraySubsegment.NAME) subsegment = xray_recorder.current_subsegment() subsegment.put_metadata( XraySubsegment.KEY, dd_trace_context, XraySubsegment.NAMESPACE ) xray_recorder.end_subsegment() else: # AWS Lambda runtime caches global variables between invocations, # reset to avoid using the context from the last invocation. dd_trace_context = {}
def upload_final_results(results): """Upload a file to S3 Parameters ---------- results: string, required Name of the local file with final results """ xray_recorder.begin_subsegment('## upload_final_results') subsegment = xray_recorder.current_subsegment() subsegment.put_metadata( 'filename', f's3://{output_bucket}/lambda-etl-refarch/output/{results}') results_path = os.path.join('/tmp', results) # upload to target S3 bucket try: response = s3.upload_file( results_path, output_bucket, 'lambda-etl-refarch/output/{}'.format(results)) log.info( f"Uploaded final results to s3://{output_bucket}/lambda-etl-refarch/output/{results}" ) subsegment.put_annotation('FINAL_RESULTS_UPLOAD', 'SUCCESS') except botocore.exceptions.ClientError as e: subsegment.put_annotation('FINAL_RESULTS_UPLOAD', 'FAILURE') log.error(f'Unable to upload final results: {results}') log.debug(e) raise xray_recorder.end_subsegment()
def xray_render(self, context): template_name = self.name or getattr(context, 'template_name', None) if template_name: name = str(template_name) subsegment = xray_recorder.current_subsegment() subsegment.name = name return Template._xray_original_render(self, context)
def get_message(): {% if cookiecutter.include_xray == "y" -%} """ You can create a sub-segment specifically to a function then capture what sub-segment that is inside your code and you can add annotations that will be indexed by X-Ray for example: put_annotation("operation", "query_db") """ # Only run xray in the AWS Lambda environment if runs_on_aws_lambda(): xray_subsegment = xray_recorder.current_subsegment() xray_subsegment.put_annotation("key", "value") {% endif -%} return { "hello": "world" }