def test_xray_context_patched(): # Could be ran in any order, so we need to tell sdk that its been unpatched xray_core_patcher._PATCHED_MODULES = set() xray_core.patch_all() xray_core.xray_recorder._context.context_missing.should.equal('LOG_ERROR') setattr(botocore.client.BaseClient, '_make_api_call', original_make_api_call) setattr(botocore.endpoint.Endpoint, '_encode_headers', original_encode_headers) setattr(requests.Session, 'request', original_session_request) setattr(requests.Session, 'prepare_request', original_session_prep_request)
def test_xray_udp_emitter_patched(): # Could be ran in any order, so we need to tell sdk that its been unpatched xray_core_patcher._PATCHED_MODULES = set() xray_core.patch_all() assert isinstance(xray_core.xray_recorder._emitter, MockEmitter) setattr(botocore.client.BaseClient, '_make_api_call', original_make_api_call) setattr(botocore.endpoint.Endpoint, '_encode_headers', original_encode_headers) setattr(requests.Session, 'request', original_session_request) setattr(requests.Session, 'prepare_request', original_session_prep_request)
def test_xray_dynamo_request_id(): # Could be ran in any order, so we need to tell sdk that its been unpatched xray_core_patcher._PATCHED_MODULES = set() xray_core.patch_all() client = boto3.client('dynamodb', region_name='us-east-1') with XRaySegment(): resp = client.list_tables() resp['ResponseMetadata'].should.contain('RequestId') id1 = resp['ResponseMetadata']['RequestId'] with XRaySegment(): client.list_tables() resp = client.list_tables() id2 = resp['ResponseMetadata']['RequestId'] id1.should_not.equal(id2) setattr(botocore.client.BaseClient, '_make_api_call', original_make_api_call) setattr(botocore.endpoint.Endpoint, '_encode_headers', original_encode_headers) setattr(requests.Session, 'request', original_session_request) setattr(requests.Session, 'prepare_request', original_session_prep_request)
import os def runs_on_aws_lambda(): """ Returns True if this function is executed on AWS Lambda service. """ return 'AWS_SAM_LOCAL' not in os.environ and 'LAMBDA_TASK_ROOT' in os.environ {%- if cookiecutter.include_xray == "y" %} from aws_xray_sdk.core import xray_recorder # Patch all supported libraries for X-Ray - More info: https://docs.aws.amazon.com/xray/latest/devguide/xray-sdk-python-patching.html if runs_on_aws_lambda(): from aws_xray_sdk.core import patch_all patch_all() {%- endif %} session = boto3.Session() def lambda_handler(event, context): """ AWS Lambda handler This method is invoked by the API Gateway: /Prod/first/{proxy+} endpoint. """ message = get_message() {% if cookiecutter.include_apigw == "y" %} return {
import json import urllib import boto3 import requests from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.core import patch_all patch_all(double_patch=True) def lambda_handler(event, context): alb = direct_alb() aga = aga_alb() return { 'statusCode': 200, 'body': json.dumps('Call global lambda via alb: ' + alb + " , via AGA: " + aga) } def direct_alb(): globalLambda = requests.get( 'http://lambda-demo-alb-1926028212.eu-west-1.elb.amazonaws.com/lambda/echo_hello' ).text.rstrip() print('direct_alb', globalLambda) return globalLambda def aga_alb(): globalLambda = requests.get(
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import os import boto3 import json from botocore import config from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.core import patch_all from MediaInsightsEngineLambdaHelper import MediaInsightsOperationHelper from MediaInsightsEngineLambdaHelper import MasExecutionError patch_all() region = os.environ["AWS_REGION"] mie_config = json.loads(os.environ['botoConfig']) config = config.Config(**mie_config) mediaconvert = boto3.client("mediaconvert", config=config, region_name=region) def lambda_handler(event, context): print("We got the following event:\n", event) operator_object = MediaInsightsOperationHelper(event) try: job_id = operator_object.metadata["MediaconvertJobId"] workflow_id = operator_object.workflow_execution_id
import os import re from datetime import datetime import boto3 from boto3.dynamodb.conditions import Key, Attr import backoff import botocore from aws_xray_sdk.core import patch_all, xray_recorder from cachetools import cached, TTLCache from lib.decorators import KinesisRecord, kinesis_handler # pylint: disable=invalid-name, line-too-long, unused-argument patch_all() # for xray tracing of boto libs dynamodb = boto3.resource('dynamodb') log = logging.getLogger() dynamo_table_cache = TTLCache(maxsize=1000, ttl=3600) json.JSONEncoder.default = lambda self, obj: (obj.isoformat() if isinstance( obj, type(datetime)) else str(obj)) S3_ARN_TO_PARTS = re.compile(r'arn:aws:s3:([^:]*):(\d*):([^/]+)/(.*)') OPENED = "opened" CLOSED = "closed" GLUE_TABLES_LOCATOR = os.environ['GLUE_TABLES_LOCATOR'] GLUE_PARTITIONS_MAPPER = os.environ['GLUE_PARTITIONS_MAPPER'] @xray_recorder.capture() @kinesis_handler(event_types=[KinesisRecord.S3_SOURCE_EVENT])
def lambda_handler(event, context): if XRAY == 'true': patch_all() get_environment_variables() print('S3_BUCKET: {}'.format(S3_BUCKET)) print('HLS_URL: {}'.format(HLS_URL)) print('HLS_URL_PLAYLIST: {}'.format(HLS_URL_PLAYLIST)) master_ttl = 3600 if (HLS_URL_PLAYLIST != urllib.unquote_plus( event['Records'][0]['s3']['object']['key'].encode('utf8'))): return 'wrong manifest' # s3_version = event['Records'][0]['s3']['object']['versionId'] # PARSE manifest print("master manifest: " + HLS_URL) # wrangle the urls base_url = 'live/' print("baseurl: {}".format(base_url)) filename_master = HLS_URL.split('/')[-1] print("filename_master: {}".format(filename_master)) ## set tmp directory tmpdir = '/tmp/' + str(uuid.uuid4()) + '/' ensure_dir(tmpdir) # GET master manifest get_s3file(S3_BUCKET, HLS_URL, tmpdir + filename_master) string_master = open(tmpdir + filename_master, 'r').read() # print("string_master: {}".format(string_master)) # PARSE frame rate for the segments segment_framerate = float( string_master.split('FRAME-RATE=')[1].split(',')[0]) print("segment_framerate: {}".format(segment_framerate)) ## Note: this is not needed, but could use it as a checksum # PARSE the m3u8 child manifestss. Returns list of these strings # filename_playlists = [x for x in string_master.split('\n') if '.m3u8' in x] # print("filename_playlists: {}".format(filename_playlists)) filename_playlist = HLS_URL_PLAYLIST.split('/')[-1] print("filename_playlist: {}".format(filename_playlist)) # GET child manifest get_s3file(S3_BUCKET, HLS_URL_PLAYLIST, tmpdir + filename_playlist) string_playlist = open(tmpdir + filename_playlist, 'r').read() # print("string_playlist: {}".format(string_playlist)) # PARSE list of DATE-time and segment name and get last one segment_datetime = '' # segment_duration = '' segment_filename = '' for x in string_playlist.split('#EXT-X-PROGRAM-DATE-TIME:'): if '.ts' in x: segment_datetime = x.split('\n#EXTINF')[0] segment_duration = x.split('#EXTINF:')[1].split(',\n')[0] segment_filename = x.split('#EXTINF:')[1].split(',\n')[1].split( '\n')[0] print('segment_datetime: {}'.format(segment_datetime)) print('segment_duration: {}'.format(segment_duration)) print('segment_filename: {}'.format(segment_filename)) # get date time string without decimal datetime_full = segment_datetime.split('.')[0] + 'Z' print('datetime_full: {}'.format(datetime_full)) # get datetime object from date time string without decimal datetime_datetime = datetime.datetime.strptime(datetime_full, "%Y-%m-%dT%H:%M:%SZ") print('datetime_datetime: {}'.format(datetime_datetime)) # get epoch of datetime epoch = datetime.datetime(1970, 1, 1) datetime_epoch = (datetime_datetime - epoch).total_seconds() print('datetime_epoch: {}'.format(datetime_epoch)) # get the decimal from the date time string datetime_decimal = segment_datetime.split('.')[1].split('Z')[:-1] print('datetime_decimal: {}'.format(datetime_decimal[0])) # convert decimal to frame number datetime_frame = int( round(int(datetime_decimal[0]) * segment_framerate / 1000, 3)) print('datetime_frame: {}'.format(datetime_frame)) # how many frames per segment frames_total = int(round(float(segment_duration) * segment_framerate, 3)) print('frames_total: {}'.format(frames_total)) # get filenames assumption started filename_base = segment_filename.split('.ts')[0] print('filename_base: {}'.format(filename_base)) # get the physical segment file get_s3file(S3_BUCKET, base_url + segment_filename, tmpdir + segment_filename) ### FFPROBE - get start PTS time output_ffprobe1 = os.popen(FFPROBE + ' ' + tmpdir + segment_filename + ' -v quiet -show_streams -of json ').read() ffprobe_json1 = json.loads(output_ffprobe1) start_time = ffprobe_json1['streams'][0]['start_time'] print('start_time: {}'.format(start_time)) ### FFPROBE - get scene change information output_ffprobe = os.popen( FFPROBE + ' -v quiet -show_streams -show_frames -of json -f lavfi "movie=' + tmpdir + segment_filename + ',select=gt(scene\,.1)"').read() ffprobe_json = json.loads(output_ffprobe) scenedetect = {} for f in ffprobe_json['frames']: this_time = float(f['pkt_pts_time']) - float(start_time) this_frame = int(round(this_time * int(segment_framerate) + 1, 0)) this_percent = int( round(float(f['tags']['lavfi.scene_score']), 2) * 100) print('scenechange \tthis_frame: {} \tthis_percent: {}%'.format( this_frame, this_percent)) scenedetect[str(this_frame)] = this_percent pprint(scenedetect) ## FFMPEG - generate a wav file wav_ffmpeg = os.popen(FFMPEG + ' -hide_banner -nostats -loglevel error -y -i ' + tmpdir + segment_filename + ' -vn -acodec pcm_s16le -ab 64k -ar 16k -ac 1 "' + tmpdir + filename_base + '.wav" > /dev/null 2>&1 ').read() ## S3 upload data = open(tmpdir + filename_base + '.wav', 'rb') pprint( s3.Bucket(S3_BUCKET).put_object(Key='audio/' + filename_base + '.wav', Body=data)) delete_file(tmpdir + filename_base + '.wav') ## TODO - trigger trainscribe lambda ## info for the whole segment dynamo_segment_object = { 'id_filename': filename_base + '-' + str(1).zfill(3) + '.jpg', 'id_type': 'segment', 'timestamp_minute': datetime_datetime.strftime("%Y-%m-%d %H:%M"), 'timestamp_second': datetime_datetime.strftime("%S"), 'timestamp_frame': datetime_frame, 'timestamp_pdt': segment_datetime, 'framerate': str(segment_framerate), 'duration': segment_duration, 'audio_file': filename_base + '.wav', 'transcribe_status': 'NOT_SUBMITTED', 'timestamp_created': int((datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds()), 'timestamp_ttl': int((datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds() + master_ttl) # 2 hours } # raise SystemExit(0) ## FFMPEG - generate a jpg for each frame output_ffmpeg = os.popen( FFMPEG + ' -hide_banner -nostats -loglevel error -y -i ' + tmpdir + segment_filename + ' -an -sn -vcodec mjpeg -pix_fmt yuvj420p -qscale 1 -b:v 2000 -bt 20M "' + tmpdir + filename_base + '-%03d.jpg" > /dev/null 2>&1 ').read() # delete ts segment myfile = tmpdir + segment_filename delete_file(myfile) scenechange_list = [] # roll file names to next second after total fps x_frame = datetime_frame x_filename = 1 ## Cycle through all frames for x in xrange(0, frames_total): if x_frame >= int(segment_framerate): x_frame = 0 datetime_datetime = datetime_datetime + datetime.timedelta( seconds=1) dynamo_epoch = (datetime_datetime - epoch).total_seconds() + (x_frame * 0.01) dynamo_minute = datetime_datetime.strftime("%Y-%m-%d %H:%M") dynamo_second = datetime_datetime.strftime("%S") dynamo_frame = str(x_frame) dynamo_filename = filename_base + '-' + str(x_filename).zfill( 3) + '.jpg' dynamo_object = {} dynamo_object = { 'id_filename': dynamo_filename, 'id_type': 'scenechange', 'timestamp_minute': dynamo_minute, 'timestamp_second': dynamo_second, 'timestamp_frame': dynamo_frame, 'timestamp_epoch': Decimal(dynamo_epoch), 'timestamp_created': int((datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds()), 'timestamp_ttl': int((datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds() + master_ttl) # 2 hours } ## Check for Scene Detection on this frame if str(x_filename) in scenedetect: ## S3 upload data = open(tmpdir + dynamo_filename, 'rb') pprint( s3.Bucket(S3_BUCKET).put_object(Key='images/' + dynamo_filename, Body=data)) dynamo_object['scenedetect'] = str(scenedetect[str(x_filename)]) # dynamo_object['scenedetect_round'] = str(int(round(scenedetect[str(x_filename)], -1))) ## Run Rekog if scene change is above 10 if scenedetect[str(x_filename)] > 10: ## REKOGNTION -- Labels print("starting rekog.... scenedetect is: " + str(scenedetect[str(x_filename)])) response = rekognition.detect_labels( Image={ "S3Object": { "Bucket": S3_BUCKET, "Name": 'images/' + dynamo_filename } }) rekog_labels_list = [] person_ok = 0 text_ok = 0 for obj in response['Labels']: rekog_labels_list.append({ 'Confidence': str(int(round(obj['Confidence']))), 'Name': obj['Name'] }) ## deltafa-summary rekog_summary_dynamo = {} rekog_summary_dynamo['scenedetect'] = str( scenedetect[str(x_filename)]) rekog_summary_dynamo['rekog_label'] = obj['Name'] rekog_summary_dynamo['rekog_type'] = 'label' rekog_summary_dynamo['id_filename'] = dynamo_filename rekog_summary_dynamo['timestamp_updated'] = int( (datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds()) rekog_summary_dynamo['timestamp_ttl'] = int( (datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds() + master_ttl) update_dyanmo_summary(rekog_summary_dynamo) ## Found Label, send over to PREKOG lambda if rekog_summary_dynamo['rekog_label'] == REKOG_LABEL: dynamo_object['rekog_label'] = obj['Name'] invoke_lambda(dynamo_object) # if int(round(obj['Confidence'])) > 49: update_dyanmo_summary(rekog_summary_dynamo) if obj['Name'] == 'Text': text_ok = 1 if obj['Name'] == 'Person': person_ok = 1 rekog_labels = sorted(rekog_labels_list, key=lambda obj: obj['Confidence']) print('rekog labels: {}'.format(rekog_labels)) dynamo_object['rekog_labels'] = rekog_labels ## REKOGNTION -- Text if text_ok == 1: print("rekog found Text, running OCR....") rekog_word_list = [] rekog_text = rekognition.detect_text( Image={ "S3Object": { "Bucket": S3_BUCKET, "Name": 'images/' + dynamo_filename } }) print('rekog words: {}'.format(rekog_text)) if rekog_text['TextDetections']: print('rekog found text in the image') for obj in rekog_text['TextDetections']: if obj['Type'] == 'WORD': rekog_word_list.append({ 'Confidence': str(int(round(obj['Confidence']))), 'Name': obj['DetectedText'] }) ## deltafa-summary rekog_summary_dynamo = {} rekog_summary_dynamo['scenedetect'] = str( scenedetect[str(x_filename)]) rekog_summary_dynamo['rekog_label'] = obj[ 'DetectedText'] rekog_summary_dynamo['rekog_type'] = 'word' rekog_summary_dynamo[ 'id_filename'] = dynamo_filename rekog_summary_dynamo[ 'timestamp_updated'] = int( (datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds()) rekog_summary_dynamo['timestamp_ttl'] = int( (datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1) ).total_seconds() + master_ttl) update_dyanmo_summary(rekog_summary_dynamo) ## Found Label, send over to PREKOG lambda if rekog_summary_dynamo['rekog_label'].upper( ) == REKOG_LABEL.upper(): dynamo_object['rekog_label'] = obj['Name'] invoke_lambda(dynamo_object) print('rekog words: {}'.format(rekog_word_list)) dynamo_object['rekog_words'] = rekog_word_list else: print('rekog did NOT find words...') ## REKOGNTION -- Celeb if person_ok == 1: print("rekog found a Person, running celeb....") rekog_celebs = rekognition.recognize_celebrities( Image={ "S3Object": { "Bucket": S3_BUCKET, "Name": 'images/' + dynamo_filename } }) if rekog_celebs['CelebrityFaces']: print("rekog found a celeb...") rekog_celebs_list = [] for obj in rekog_celebs['CelebrityFaces']: # invoke_lambda( rekog_label_dynamo ) ## deltafa-summary rekog_summary_dynamo = {} rekog_summary_dynamo['scenedetect'] = str( scenedetect[str(x_filename)]) rekog_summary_dynamo['rekog_label'] = obj['Name'] rekog_summary_dynamo['rekog_type'] = 'celeb' rekog_summary_dynamo[ 'id_filename'] = dynamo_filename rekog_summary_dynamo['timestamp_updated'] = int(( datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)).total_seconds()) rekog_summary_dynamo['timestamp_ttl'] = int( (datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1) ).total_seconds() + master_ttl) update_dyanmo_summary(rekog_summary_dynamo) print('rekgo celeb: {}'.format(rekog_celebs_list)) dynamo_object['rekog_celebs'] = rekog_celebs_list else: print("rekog did NOT find celeb...") # PUT DYNAMO - main put_dynamo_main(dynamo_object) scenechange_list.append(dynamo_object['id_filename']) # clean up jpgs delete_file(tmpdir + dynamo_filename) x_frame += 1 x_filename += 1 # full segment information dynamo_segment_object['scenechange_list'] = scenechange_list pprint(dynamo_segment_object) put_dynamo_main(dynamo_segment_object) write_json() return 'SUCCESS: it ran'
def enable_xray(): patch_all()
def create_app(): app = Flask(__name__) # Configure xray tracing xray_recorder.configure(service='dashboard') XRayMiddleware(app, xray_recorder) patch_all() # Setup logging setup_logging(log_levels.get(os.environ.get("LOG_LEVEL", "INFO"))) log = logging.getLogger(__name__) @app.errorhandler(HTTPException) def handle_error(error): error_dict = { "message": str(error) if isinstance(error, HTTPException) else '', "exc_type": type(error).__name__, "status_code": error.code } log.exception(error) return make_response( json.dumps({ "message": "Error occured", "code": error.code, "trace_id": get_trace_id(request.headers.get("X-Amzn-Trace-Id")), "request_id": get_request_id() }), 500) @app.route("/") def hello(): try: resp = requests.get("{}/".format(COUNTER_ENDPOINT), headers={"X-Request-Id": get_request_id()}) log.info("Received request, trace_id={}, req_id={}".format( get_trace_id(request.headers.get("X-Amzn-Trace-Id")), get_request_id())) if resp.status_code == 200: resp = json.loads(resp.text) resp = json.dumps({ "message": "Counter is reachable", "count": resp["count"], "counter_service_id": resp["counter_service_id"], "dashboard_service_id": socket.gethostname(), "trace_id": get_trace_id(request.headers.get("X-Amzn-Trace-Id")), "request_id": get_request_id() }) else: log.info( "Error calling {} service! code={}, service={}".format( resp.text, resp.status_code, COUNTER_ENDPOINT)) resp = make_response( json.dumps({ "message": "Error calling %s service" % COUNTER_ENDPOINT, "code": resp.status_code, "error": resp.text, "trace_id": get_trace_id(request.headers.get("X-Amzn-Trace-Id")), "request_id": get_request_id() }), 500) except requests.exceptions.RequestException as ex: log.info("Error connecting to counter at %s, %s" % (COUNTER_ENDPOINT, str(ex))) resp = make_response( json.dumps({ "message": str(ex), "trace_id": get_trace_id(request.headers.get("X-Amzn-Trace-Id")), "request_id": get_request_id() }), 500) return resp @app.route("/fail") def fail(): code = request.args.get('code') log.info( "/fail?code={} called! Relaying to counter service, trace_id={}, req_id={}" .format(code, get_trace_id(request.headers.get("X-Amzn-Trace-Id")), get_request_id())) resp = requests.get("{}/fail?code={}".format(COUNTER_ENDPOINT, code), headers={"X-Request-Id": get_request_id()}) resp = make_response( json.dumps({ "message": "Received code={} on {}/fail?code={}".format( resp.status_code, COUNTER_ENDPOINT, code) }), int(resp.status_code)) return resp @app.route("/health") def health(): return json.dumps({"status": "DASHBOARD_HEALTHY"}) return app
# required for serverless framework to un-compress binaries import json import os import sys import logging import re logger = logging.getLogger() logger.setLevel(logging.DEBUG) import boto3 from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.core import patch_all patch_all( ) # This will patch boto3 and any other supported client library to emit xray data. # Credentails for the lambdas are set via IAM roles in the serverless.yml file. lambda_client = boto3.client("lambda") # client object to access lambda APIs sqs_client = boto3.client("sqs") # Let's figure out what stage we are running and set the logging and xray configuration based on it. DEPLOYMENT_STAGE = os.environ.get("DEPLOYMENT_STAGE") if DEPLOYMENT_STAGE is None: logger.debug("No DEPLOYMENT_STAGE environment variable set. Assuming dev.") DEPLOYMENT_STAGE = ( "dev" # The default stage will be dev even if nothing is specified. ) # Next I want to set a specific xray sampling configurarion.
def lambda_handler(event, context): if XRAY == 'true': patch_all() get_environment_variables() timestamp_start = (datetime.datetime.now() - datetime.timedelta(seconds=60)) timestamp_end = datetime.datetime.now() timestamps = { 'start' : "{:%b %d, %H:%M:%S}".format(timestamp_start), 'end' : "{:%b %d, %H:%M:%S}".format(timestamp_end), } print('timestamps: {}'.format(timestamps)) full_list = [] images_list = {} rekog_type = 'label' this_response = get_dynamo_summary(rekog_type) for stuff in this_response: full_list.append(stuff['rekog_label'].lower()) # images_list.append({stuff['rekog_label'].lower() : stuff['id_filename']}) images_list[stuff['rekog_label'].lower()] = stuff['id_filename'] rekog_type = 'word' this_response = get_dynamo_summary(rekog_type) for stuff in this_response: full_list.append(stuff['rekog_label'].lower()) # images_list.append({stuff['rekog_label'].lower() : stuff['id_filename']}) images_list[stuff['rekog_label'].lower()] = stuff['id_filename'] print('full list: {}'.format(full_list)) print('images list: {}'.format(images_list)) # boulder eats: [u'girl', u'salad', u'human', u'woman', u'blonde', u'food', u'dinner', u'face', u'bowl', u'supper', u'lunch', u'female', u'person', u'meal', u'people', u'portrait', u'sport', u'sports', u'working out', u'produce', u'exercise', u'long sleeve', u'dahlia', u'sprout', u'daisy', u'clothing', u'flora', u'ornament', u'daisies', u'flower'] # top gear: [u'human', u'vehicle', u'transportation', u'face', u'automobile', u'driving', u'person', u'people', u'portrait', u'car', u'convertible', u'apparel', u'shoe', u'clothing', u'footwear', u'wedge', u'triangle', u'laptop', u'pc', u'computer', u'electronics', u'bumper', u'white board', u'mold', u'adorable', u'sink', u'coupe', u'sports car', u'arm', u'airport'] # top gear 2: [u'human', u'vehicle', u'transportation', u'face', u'automobile', u'driving', u'person', u'people', u'portrait', u'car', u'convertible', u'apparel', u'shoe', u'clothing', u'footwear', u'wedge', u'triangle', u'laptop', u'pc', u'computer', u'electronics', u'bumper', u'white board', u'mold', u'adorable', u'sink', u'coupe', u'sports car', u'arm', u'airport'] # zoo bear: [u'mammal', u'hole', u'bear', u'wildlife', u'animal', u'zoo', u'astronomy', u'space', u'universe', u'outer space'] # hockey: [u'skating', u'human', u'team sport', u'hockey', u'sport', u'ice hockey', u'team', u'ice skating', u'sports', u'person', u'rink', u'people', u'modern art', u'art'] match_list = { 'formula_racing' : ['road', 'street', 'vehicle', 'wheel' ], 'moods_tour' : ['instrument', 'band', 'guitar', 'city', 'alcohol'], 'odeur' : ['clothing', 'flora'], 'robots' : ['toys', 'electronics'], 'save_polarbear' : ['animal', 'bear', 'cat', 'dog'], 'sportz' : ['baseball', 'football', 'basketball', 'hockey', 'sports'], 'visit_alps' : ['ski', 'snow', 'powder', 'luggage',], 'visit_caribbean' : ['beach', 'sun', 'drinks', 'nature', 'weather', 'sunset' ], 'g_i_joe' : ['toy', 'sport', 'outdoors'], 'candy_bar' : ['salad'], 'video_game' : ['watch', 'television', 'electronics'], 'super_boy' : ['grass', 'outer space', ], 'toy_puppy' : ['toy', 'dog', 'zoo', 'bird'], 'action_figure' : ['toy', 'gun', 'motorcycle', ], # 'toy_car' : ['toy', 'car', 'electronics'], 'chocoloate_drink' : ['drinks', 'food', 'lunch'], # 'soft_drink' : ['drinks', 'glass', 'beverage'], } thirty_secs = [ 'g_i_joe', 'candy_bar', 'video_game', 'super_boy', 'toy_puppy', 'action_figure', 'toy_car', 'chocoloate_drink', 'soft_drink', ] ad_playlist = [] ad_matches = [] images_matches = {} for ad_type in match_list: for ad_label in match_list[ad_type]: if ad_label in full_list: print('MATCHED! type: {} \tlabel: {}'.format(ad_type,ad_label)) ad_playlist.append(ad_type) ad_matches.append({ad_type : ad_label}) images_matches[ad_label] = {'image' : images_list[ad_label], 'type': ad_type } print('images_matches: {}'.format(images_matches)) # pprint(ad_playlist) if len(ad_playlist) < 1: ad_playlist.append('default') while len(ad_playlist) < 5: ad_playlist.extend(ad_playlist) print('playlist lineup: {}'.format(ad_playlist)) full_xml = '''<VAST version="3.0"> ''' x = 0 for ad in ad_playlist: duration = '00:00:15' if ad in thirty_secs: duration = '00:00:30' x += 1 full_xml += ''' <Ad sequence="''' + str(x) + '''"> <InLine> <AdSystem>2.0</AdSystem> <AdTitle>ad-''' + str(x) + '''</AdTitle> <Impression></Impression> <Creatives> <Creative id="''' + str(ad) + '''-3"> <Linear> <Duration>''' + str(duration) + '''</Duration> <TrackingEvents></TrackingEvents> <MediaFiles> <MediaFile delivery="progressive" type="video/mp4" width="1920" height="1080"> <![CDATA[ https://s3.amazonaws.com/ces2018-demo/ads/''' + ad + '''-3.mp4 ]]> </MediaFile> </MediaFiles> </Linear> </Creative> </Creatives> </InLine> </Ad> ''' full_xml += ''' </VAST>''' # print('xml: {}'.format(full_xml)) json_together = { # 'full_list' : full_list, 'matches' : ad_matches, 'images_list' : images_list, 'images_matches' : images_matches, 'timestamps' : timestamps, } # pprint(json_together) json_string = json.dumps(json_together, cls=DecimalEncoder) dynamo_filename = 'list-ads.json' ## set tmp directory tmpdir = '/tmp/' + str(uuid.uuid4()) + '/' ensure_dir(tmpdir) with open(tmpdir + dynamo_filename, 'w') as outfile: outfile.write(json_string) ## S3 upload data = open(tmpdir + dynamo_filename, 'rb') s3 = boto3.resource('s3') pprint(s3.Bucket(S3_BUCKET).put_object(Key=dynamo_filename, Body=data)) return { 'statusCode': '200', # 'body': re.sub(r'(.+m3u8|.+ts)', base_url + r'\1', master,re.M), 'body': full_xml, 'headers': {'Content-Type': 'application/xml', 'Access-Control-Allow-Origin': '*'} }
def lambda_handler(event, context): if XRAY == 'true': patch_all() ### SNS from MediaConvert if 'detail-type' in event: if event['detail-type'] == 'MediaConvert Job State Change': emc_list = { 'label' : event['detail']['userMetadata']['label'], 'label_sort' : event['detail']['userMetadata']['label_sort'], 'emc_url' : event['detail']['userMetadata']['label_sort'] + '_video.mp4' , 'emc_status' : event['detail']['status'], } update_dyanmo_list ( emc_list ) else: # set here because of global variable mediaconvert = boto3.client('mediaconvert', endpoint_url=EMC_ENDPOINT) master_ttl = 3600 url_full = event['emp_url'] s3_bucket = event['s3_bucket'] event_id = event['label_sort'] # PARSE manifest print("master manifest: " + url_full) # wrangle the urls base_url = '/'.join(url_full.split('/')[:-1]) + '/' print("baseurl: {}".format(base_url)) filename_master = url_full.split('/')[-1] print("filename_master: {}".format(filename_master)) # GET master manifest string_master = get_url(base_url + filename_master) print("string_master: {}".format(string_master)) # PARSE the m3u8 child manifestss. Returns list of these strings filename_playlists = [x for x in string_master.split('\n') if '.m3u8' in x] print("filename_playlists: {}".format(filename_playlists)) # GET child manifest string_playlist = get_url(base_url + filename_playlists[0]) print("string_playlist: {}".format(string_playlist)) # get filenames assumption started # filename_base = segment_filename.split('.ts')[0] # print('filename_base: {}'.format(filename_base)) ## set tmp directory tmpdir = '/tmp/' + event_id + '/' ensure_dir(tmpdir) print('tmpdir: {}'.format(tmpdir)) # get the physical master manifest file save_file(tmpdir + filename_master.split('?')[0], get_url(base_url + filename_master)) data = open(tmpdir + filename_master.split('?')[0], 'rb') pprint(s3.Bucket(s3_bucket).put_object(Key='video' + tmpdir + filename_master.split('?')[0], Body=data)) delete_file(tmpdir + filename_master.split('?')[0]) # get the physical child manifest file save_file(tmpdir + filename_playlists[0].split('?')[0], get_url(base_url + filename_playlists[0])) data = open(tmpdir + filename_playlists[0].split('?')[0], 'rb') pprint(s3.Bucket(s3_bucket).put_object(Key='video' + tmpdir + filename_playlists[0].split('?')[0], Body=data)) delete_file(tmpdir + filename_playlists[0].split('?')[0]) # get the physical segment file for x in string_playlist.split('\n'): if '.ts' in x: pprint(x) segment_filename = x.split('?')[0] save_file(tmpdir + segment_filename, get_url(base_url + segment_filename)) data = open(tmpdir + segment_filename, 'rb') pprint(s3.Bucket(s3_bucket).put_object(Key='video' + tmpdir + segment_filename, Body=data)) delete_file(tmpdir + segment_filename) ## MediaConvert # mediaConvertRole = "arn:aws:iam::1234567890:role/vod-MediaConvertIAMRole-1BIVDOODMZT1L-MediaConvertRole" # mediaConvertEndpoint = "https://vweb51sm.mediaconvert.us-west-2.amazonaws.com" time_start = str(event['time_start'].split(' ')[1]) + ":" + event['time_start_frame'].zfill(2) time_end = str(event['time_end'].split(' ')[1]) + ":" + event['time_end_frame'].zfill(2) # time_start = event['time_start'].split(' ')[1] # time_end = event['time_end'].split(' ')[1] # mediaconvert = boto3.client('mediaconvert', endpoint_url=EMC_ENDPOINT) jobMetadata = {'label_sort': str(event['label_sort']), 'label': str(event['label']) } jobSettings = { "OutputGroups": [ { "Name": "File Group", "Outputs": [ { "ContainerSettings": { "Container": "MP4", "Mp4Settings": { "CslgAtom": "INCLUDE", "FreeSpaceBox": "EXCLUDE", "MoovPlacement": "PROGRESSIVE_DOWNLOAD" } }, "VideoDescription": { "Width": 1280, "ScalingBehavior": "DEFAULT", "Height": 720, "TimecodeInsertion": "DISABLED", "AntiAlias": "ENABLED", "Sharpness": 50, "CodecSettings": { "Codec": "H_264", "H264Settings": { "InterlaceMode": "PROGRESSIVE", "NumberReferenceFrames": 3, "Syntax": "DEFAULT", "Softness": 0, "GopClosedCadence": 1, "GopSize": 90, "Slices": 1, "GopBReference": "DISABLED", "SlowPal": "DISABLED", "SpatialAdaptiveQuantization": "ENABLED", "TemporalAdaptiveQuantization": "ENABLED", "FlickerAdaptiveQuantization": "DISABLED", "EntropyEncoding": "CABAC", "Bitrate": 5000000, "FramerateControl": "INITIALIZE_FROM_SOURCE", "RateControlMode": "CBR", "CodecProfile": "MAIN", "Telecine": "NONE", "MinIInterval": 0, "AdaptiveQuantization": "HIGH", "CodecLevel": "AUTO", "FieldEncoding": "PAFF", "SceneChangeDetect": "ENABLED", "QualityTuningLevel": "SINGLE_PASS", "FramerateConversionAlgorithm": "DUPLICATE_DROP", "UnregisteredSeiTimecode": "DISABLED", "GopSizeUnits": "FRAMES", "ParControl": "INITIALIZE_FROM_SOURCE", "NumberBFramesBetweenReferenceFrames": 2, "RepeatPps": "DISABLED" } }, "AfdSignaling": "NONE", "DropFrameTimecode": "ENABLED", "RespondToAfd": "NONE", "ColorMetadata": "INSERT" }, "AudioDescriptions": [ { "AudioTypeControl": "FOLLOW_INPUT", "CodecSettings": { "Codec": "AAC", "AacSettings": { "AudioDescriptionBroadcasterMix": "NORMAL", "Bitrate": 96000, "RateControlMode": "CBR", "CodecProfile": "LC", "CodingMode": "CODING_MODE_2_0", "RawFormat": "NONE", "SampleRate": 48000, "Specification": "MPEG4" } }, "LanguageCodeControl": "FOLLOW_INPUT" } ], "NameModifier": "_video" } ], "OutputGroupSettings": { "Type": "FILE_GROUP_SETTINGS", "FileGroupSettings": { "Destination": "s3://" + s3_bucket + "/video/" + event_id } } } ], "AdAvailOffset": 0, "Inputs": [ { "InputClippings": [ { "EndTimecode": time_end, "StartTimecode": time_start } ], "AudioSelectors": { "Audio Selector 1": { "Offset": 0, "DefaultSelection": "DEFAULT", "ProgramSelection": 1 } }, "VideoSelector": { "ColorSpace": "FOLLOW" }, "FilterEnable": "AUTO", "PsiControl": "USE_PSI", "FilterStrength": 0, "DeblockFilter": "DISABLED", "DenoiseFilter": "DISABLED", "TimecodeSource": "EMBEDDED", "FileInput": "s3://" + s3_bucket + "/video/tmp/" + event_id + "/index.m3u8" } ] } pprint(jobSettings) job = mediaconvert.create_job(Role=EMC_ROLE, UserMetadata=jobMetadata, Settings=jobSettings) # pprint(job) return 'SUCCESS: it ran'
BOTO_RESOURCE_KWARGS={"region_name": aws_region}, CELL_SETS_BUCKET=f"cell-sets-{cluster_env}", SOURCE_BUCKET=f"processed-matrix-{cluster_env}", RESULTS_BUCKET=f"worker-results-{cluster_env}", R_WORKER_URL="http://localhost:4000", # this works because in CI, `data/` is deployed under `worker/` # whereas in a container, it is mounted to `/data`. Either way, this ensures # that the appropriate path is selected, as both are two directories up LOCAL_DIR=os.path.join(os.pardir, os.pardir, "data"), ) config.API_URL = ( f"http://api-{config.SANDBOX_ID}.api-{config.SANDBOX_ID}.svc.cluster.local:3000" ) if cluster_env == "development" or cluster_env == "test": config.AWS_ACCOUNT_ID = "000000000000" config.BOTO_RESOURCE_KWARGS["aws_access_key_id"] = "my-key" config.BOTO_RESOURCE_KWARGS["aws_secret_access_key"] = "my-secret-key" config.API_URL = "http://host.docker.internal:3000" global_sdk_config.set_sdk_enabled(False) if cluster_env == "development": config.BOTO_RESOURCE_KWARGS[ "endpoint_url"] = "http://host.docker.internal:4566" config.R_WORKER_URL = "http://r:4000" if cluster_env != "test": core.patch_all()
def lambda_handler(event, context): if XRAY == 'true': patch_all() get_environment_variables() print('event: {}'.format(event)) print('S3_BUCKET: {}'.format(S3_BUCKET)) print('EMP_URL: {}'.format(EMP_URL)) if EMP_URL == 'not-set': return 'ERROR: EMP_URL was not set' rekog_label = event['rekog_label'] id_filename = event['id_filename'] print('rekog_label: {} \tid_filename: {}'.format(rekog_label, id_filename)) ## BACKWARDS in time if int(event['scenedetect']) > 50: print('first scene is {}, skipping hops_backwards'.format( event['scenedetect'])) hops_backward = { 'last_hop': event, 'messages': [ 'first scene is {}, skipping hops_backwards'.format( event['scenedetect']) ], 'all_hops': { '1': event }, } else: hops_backward = detect_hops(rekog_label, id_filename, False) print('hops_backward: {}'.format(hops_backward)) ## FORWARD in time ( may have to wait for time to actually happen ) hops_forward = detect_hops(rekog_label, id_filename, True) print('hops_forward: {}'.format(hops_forward)) query_dict = {} query_dict['label_image'] = id_filename query_dict['label'] = rekog_label query_dict['time_start'] = hops_backward['last_hop'][ 'timestamp_minute'] + ':' + hops_backward['last_hop'][ 'timestamp_second'] query_dict['time_start_frame'] = str( hops_backward['last_hop']['timestamp_frame']) query_dict['time_start_image'] = hops_backward['last_hop']['id_filename'] query_dict['messages_backward'] = hops_backward['messages'] query_dict['hops_backward'] = hops_backward['all_hops'] query_dict['time_end'] = hops_forward['last_hop'][ 'timestamp_minute'] + ':' + hops_forward['last_hop']['timestamp_second'] query_dict['time_end_frame'] = str( hops_forward['last_hop']['timestamp_frame']) query_dict['time_end_image'] = hops_forward['last_hop']['id_filename'] query_dict['messages_forward'] = hops_forward['messages'] query_dict['hops_forward'] = hops_forward['all_hops'] # pprint(query_dict) query_dict[ 'emp_url'] = EMP_URL + '?start=' + query_dict['time_start'].replace( ' ', 'T') + '+00:00&end=' + query_dict['time_end'].replace( ' ', 'T') + '+00:00' lambda_event = put_dynamo(query_dict) pprint(lambda_event) invoke_lambda(lambda_event) return 'SUCCESS: it ran'