def test_xray_context_patched():
# Could be ran in any order, so we need to tell sdk that its been unpatched
xray_core_patcher._PATCHED_MODULES = set()
xray_core.patch_all()
xray_core.xray_recorder._context.context_missing.should.equal('LOG_ERROR')
setattr(botocore.client.BaseClient, '_make_api_call', original_make_api_call)
setattr(botocore.endpoint.Endpoint, '_encode_headers', original_encode_headers)
setattr(requests.Session, 'request', original_session_request)
setattr(requests.Session, 'prepare_request', original_session_prep_request)
def test_xray_udp_emitter_patched():
# Could be ran in any order, so we need to tell sdk that its been unpatched
xray_core_patcher._PATCHED_MODULES = set()
xray_core.patch_all()
assert isinstance(xray_core.xray_recorder._emitter, MockEmitter)
setattr(botocore.client.BaseClient, '_make_api_call', original_make_api_call)
setattr(botocore.endpoint.Endpoint, '_encode_headers', original_encode_headers)
setattr(requests.Session, 'request', original_session_request)
setattr(requests.Session, 'prepare_request', original_session_prep_request)
def test_xray_dynamo_request_id():
# Could be ran in any order, so we need to tell sdk that its been unpatched
xray_core_patcher._PATCHED_MODULES = set()
xray_core.patch_all()
client = boto3.client('dynamodb', region_name='us-east-1')
with XRaySegment():
resp = client.list_tables()
resp['ResponseMetadata'].should.contain('RequestId')
id1 = resp['ResponseMetadata']['RequestId']
with XRaySegment():
client.list_tables()
resp = client.list_tables()
id2 = resp['ResponseMetadata']['RequestId']
id1.should_not.equal(id2)
setattr(botocore.client.BaseClient, '_make_api_call', original_make_api_call)
setattr(botocore.endpoint.Endpoint, '_encode_headers', original_encode_headers)
setattr(requests.Session, 'request', original_session_request)
setattr(requests.Session, 'prepare_request', original_session_prep_request)
import os
def runs_on_aws_lambda():
"""
Returns True if this function is executed on AWS Lambda service.
"""
return 'AWS_SAM_LOCAL' not in os.environ and 'LAMBDA_TASK_ROOT' in os.environ
{%- if cookiecutter.include_xray == "y" %}
from aws_xray_sdk.core import xray_recorder
# Patch all supported libraries for X-Ray - More info: https://docs.aws.amazon.com/xray/latest/devguide/xray-sdk-python-patching.html
if runs_on_aws_lambda():
from aws_xray_sdk.core import patch_all
patch_all()
{%- endif %}
session = boto3.Session()
def lambda_handler(event, context):
"""
AWS Lambda handler
This method is invoked by the API Gateway: /Prod/first/{proxy+} endpoint.
"""
message = get_message()
{% if cookiecutter.include_apigw == "y" %}
return {
import json
import urllib
import boto3
import requests
from aws_xray_sdk.core import xray_recorder
from aws_xray_sdk.core import patch_all
patch_all(double_patch=True)
def lambda_handler(event, context):
alb = direct_alb()
aga = aga_alb()
return {
'statusCode':
200,
'body':
json.dumps('Call global lambda via alb: ' + alb + " , via AGA: " + aga)
}
def direct_alb():
globalLambda = requests.get(
'http://lambda-demo-alb-1926028212.eu-west-1.elb.amazonaws.com/lambda/echo_hello'
).text.rstrip()
print('direct_alb', globalLambda)
return globalLambda
def aga_alb():
globalLambda = requests.get(
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
import os
import boto3
import json
from botocore import config
from aws_xray_sdk.core import xray_recorder
from aws_xray_sdk.core import patch_all
from MediaInsightsEngineLambdaHelper import MediaInsightsOperationHelper
from MediaInsightsEngineLambdaHelper import MasExecutionError
patch_all()
region = os.environ["AWS_REGION"]
mie_config = json.loads(os.environ['botoConfig'])
config = config.Config(**mie_config)
mediaconvert = boto3.client("mediaconvert", config=config, region_name=region)
def lambda_handler(event, context):
print("We got the following event:\n", event)
operator_object = MediaInsightsOperationHelper(event)
try:
job_id = operator_object.metadata["MediaconvertJobId"]
workflow_id = operator_object.workflow_execution_id
import os
import re
from datetime import datetime
import boto3
from boto3.dynamodb.conditions import Key, Attr
import backoff
import botocore
from aws_xray_sdk.core import patch_all, xray_recorder
from cachetools import cached, TTLCache
from lib.decorators import KinesisRecord, kinesis_handler
# pylint: disable=invalid-name, line-too-long, unused-argument
patch_all() # for xray tracing of boto libs
dynamodb = boto3.resource('dynamodb')
log = logging.getLogger()
dynamo_table_cache = TTLCache(maxsize=1000, ttl=3600)
json.JSONEncoder.default = lambda self, obj: (obj.isoformat() if isinstance(
obj, type(datetime)) else str(obj))
S3_ARN_TO_PARTS = re.compile(r'arn:aws:s3:([^:]*):(\d*):([^/]+)/(.*)')
OPENED = "opened"
CLOSED = "closed"
GLUE_TABLES_LOCATOR = os.environ['GLUE_TABLES_LOCATOR']
GLUE_PARTITIONS_MAPPER = os.environ['GLUE_PARTITIONS_MAPPER']
@xray_recorder.capture()
@kinesis_handler(event_types=[KinesisRecord.S3_SOURCE_EVENT])
def lambda_handler(event, context):
if XRAY == 'true':
patch_all()
get_environment_variables()
print('S3_BUCKET: {}'.format(S3_BUCKET))
print('HLS_URL: {}'.format(HLS_URL))
print('HLS_URL_PLAYLIST: {}'.format(HLS_URL_PLAYLIST))
master_ttl = 3600
if (HLS_URL_PLAYLIST != urllib.unquote_plus(
event['Records'][0]['s3']['object']['key'].encode('utf8'))):
return 'wrong manifest'
# s3_version = event['Records'][0]['s3']['object']['versionId']
# PARSE manifest
print("master manifest: " + HLS_URL)
# wrangle the urls
base_url = 'live/'
print("baseurl: {}".format(base_url))
filename_master = HLS_URL.split('/')[-1]
print("filename_master: {}".format(filename_master))
## set tmp directory
tmpdir = '/tmp/' + str(uuid.uuid4()) + '/'
ensure_dir(tmpdir)
# GET master manifest
get_s3file(S3_BUCKET, HLS_URL, tmpdir + filename_master)
string_master = open(tmpdir + filename_master, 'r').read()
# print("string_master: {}".format(string_master))
# PARSE frame rate for the segments
segment_framerate = float(
string_master.split('FRAME-RATE=')[1].split(',')[0])
print("segment_framerate: {}".format(segment_framerate))
## Note: this is not needed, but could use it as a checksum
# PARSE the m3u8 child manifestss. Returns list of these strings
# filename_playlists = [x for x in string_master.split('\n') if '.m3u8' in x]
# print("filename_playlists: {}".format(filename_playlists))
filename_playlist = HLS_URL_PLAYLIST.split('/')[-1]
print("filename_playlist: {}".format(filename_playlist))
# GET child manifest
get_s3file(S3_BUCKET, HLS_URL_PLAYLIST, tmpdir + filename_playlist)
string_playlist = open(tmpdir + filename_playlist, 'r').read()
# print("string_playlist: {}".format(string_playlist))
# PARSE list of DATE-time and segment name and get last one
segment_datetime = ''
# segment_duration = ''
segment_filename = ''
for x in string_playlist.split('#EXT-X-PROGRAM-DATE-TIME:'):
if '.ts' in x:
segment_datetime = x.split('\n#EXTINF')[0]
segment_duration = x.split('#EXTINF:')[1].split(',\n')[0]
segment_filename = x.split('#EXTINF:')[1].split(',\n')[1].split(
'\n')[0]
print('segment_datetime: {}'.format(segment_datetime))
print('segment_duration: {}'.format(segment_duration))
print('segment_filename: {}'.format(segment_filename))
# get date time string without decimal
datetime_full = segment_datetime.split('.')[0] + 'Z'
print('datetime_full: {}'.format(datetime_full))
# get datetime object from date time string without decimal
datetime_datetime = datetime.datetime.strptime(datetime_full,
"%Y-%m-%dT%H:%M:%SZ")
print('datetime_datetime: {}'.format(datetime_datetime))
# get epoch of datetime
epoch = datetime.datetime(1970, 1, 1)
datetime_epoch = (datetime_datetime - epoch).total_seconds()
print('datetime_epoch: {}'.format(datetime_epoch))
# get the decimal from the date time string
datetime_decimal = segment_datetime.split('.')[1].split('Z')[:-1]
print('datetime_decimal: {}'.format(datetime_decimal[0]))
# convert decimal to frame number
datetime_frame = int(
round(int(datetime_decimal[0]) * segment_framerate / 1000, 3))
print('datetime_frame: {}'.format(datetime_frame))
# how many frames per segment
frames_total = int(round(float(segment_duration) * segment_framerate, 3))
print('frames_total: {}'.format(frames_total))
# get filenames assumption started
filename_base = segment_filename.split('.ts')[0]
print('filename_base: {}'.format(filename_base))
# get the physical segment file
get_s3file(S3_BUCKET, base_url + segment_filename,
tmpdir + segment_filename)
### FFPROBE - get start PTS time
output_ffprobe1 = os.popen(FFPROBE + ' ' + tmpdir + segment_filename +
' -v quiet -show_streams -of json ').read()
ffprobe_json1 = json.loads(output_ffprobe1)
start_time = ffprobe_json1['streams'][0]['start_time']
print('start_time: {}'.format(start_time))
### FFPROBE - get scene change information
output_ffprobe = os.popen(
FFPROBE +
' -v quiet -show_streams -show_frames -of json -f lavfi "movie=' +
tmpdir + segment_filename + ',select=gt(scene\,.1)"').read()
ffprobe_json = json.loads(output_ffprobe)
scenedetect = {}
for f in ffprobe_json['frames']:
this_time = float(f['pkt_pts_time']) - float(start_time)
this_frame = int(round(this_time * int(segment_framerate) + 1, 0))
this_percent = int(
round(float(f['tags']['lavfi.scene_score']), 2) * 100)
print('scenechange \tthis_frame: {} \tthis_percent: {}%'.format(
this_frame, this_percent))
scenedetect[str(this_frame)] = this_percent
pprint(scenedetect)
## FFMPEG - generate a wav file
wav_ffmpeg = os.popen(FFMPEG +
' -hide_banner -nostats -loglevel error -y -i ' +
tmpdir + segment_filename +
' -vn -acodec pcm_s16le -ab 64k -ar 16k -ac 1 "' +
tmpdir + filename_base +
'.wav" > /dev/null 2>&1 ').read()
## S3 upload
data = open(tmpdir + filename_base + '.wav', 'rb')
pprint(
s3.Bucket(S3_BUCKET).put_object(Key='audio/' + filename_base + '.wav',
Body=data))
delete_file(tmpdir + filename_base + '.wav')
## TODO - trigger trainscribe lambda
## info for the whole segment
dynamo_segment_object = {
'id_filename':
filename_base + '-' + str(1).zfill(3) + '.jpg',
'id_type':
'segment',
'timestamp_minute':
datetime_datetime.strftime("%Y-%m-%d %H:%M"),
'timestamp_second':
datetime_datetime.strftime("%S"),
'timestamp_frame':
datetime_frame,
'timestamp_pdt':
segment_datetime,
'framerate':
str(segment_framerate),
'duration':
segment_duration,
'audio_file':
filename_base + '.wav',
'transcribe_status':
'NOT_SUBMITTED',
'timestamp_created':
int((datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1)).total_seconds()),
'timestamp_ttl':
int((datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1)).total_seconds() +
master_ttl) # 2 hours
}
# raise SystemExit(0)
## FFMPEG - generate a jpg for each frame
output_ffmpeg = os.popen(
FFMPEG + ' -hide_banner -nostats -loglevel error -y -i ' + tmpdir +
segment_filename +
' -an -sn -vcodec mjpeg -pix_fmt yuvj420p -qscale 1 -b:v 2000 -bt 20M "'
+ tmpdir + filename_base + '-%03d.jpg" > /dev/null 2>&1 ').read()
# delete ts segment
myfile = tmpdir + segment_filename
delete_file(myfile)
scenechange_list = []
# roll file names to next second after total fps
x_frame = datetime_frame
x_filename = 1
## Cycle through all frames
for x in xrange(0, frames_total):
if x_frame >= int(segment_framerate):
x_frame = 0
datetime_datetime = datetime_datetime + datetime.timedelta(
seconds=1)
dynamo_epoch = (datetime_datetime - epoch).total_seconds() + (x_frame *
0.01)
dynamo_minute = datetime_datetime.strftime("%Y-%m-%d %H:%M")
dynamo_second = datetime_datetime.strftime("%S")
dynamo_frame = str(x_frame)
dynamo_filename = filename_base + '-' + str(x_filename).zfill(
3) + '.jpg'
dynamo_object = {}
dynamo_object = {
'id_filename':
dynamo_filename,
'id_type':
'scenechange',
'timestamp_minute':
dynamo_minute,
'timestamp_second':
dynamo_second,
'timestamp_frame':
dynamo_frame,
'timestamp_epoch':
Decimal(dynamo_epoch),
'timestamp_created':
int((datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1)).total_seconds()),
'timestamp_ttl':
int((datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1)).total_seconds() +
master_ttl) # 2 hours
}
## Check for Scene Detection on this frame
if str(x_filename) in scenedetect:
## S3 upload
data = open(tmpdir + dynamo_filename, 'rb')
pprint(
s3.Bucket(S3_BUCKET).put_object(Key='images/' +
dynamo_filename,
Body=data))
dynamo_object['scenedetect'] = str(scenedetect[str(x_filename)])
# dynamo_object['scenedetect_round'] = str(int(round(scenedetect[str(x_filename)], -1)))
## Run Rekog if scene change is above 10
if scenedetect[str(x_filename)] > 10:
## REKOGNTION -- Labels
print("starting rekog.... scenedetect is: " +
str(scenedetect[str(x_filename)]))
response = rekognition.detect_labels(
Image={
"S3Object": {
"Bucket": S3_BUCKET,
"Name": 'images/' + dynamo_filename
}
})
rekog_labels_list = []
person_ok = 0
text_ok = 0
for obj in response['Labels']:
rekog_labels_list.append({
'Confidence':
str(int(round(obj['Confidence']))),
'Name':
obj['Name']
})
## deltafa-summary
rekog_summary_dynamo = {}
rekog_summary_dynamo['scenedetect'] = str(
scenedetect[str(x_filename)])
rekog_summary_dynamo['rekog_label'] = obj['Name']
rekog_summary_dynamo['rekog_type'] = 'label'
rekog_summary_dynamo['id_filename'] = dynamo_filename
rekog_summary_dynamo['timestamp_updated'] = int(
(datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1)).total_seconds())
rekog_summary_dynamo['timestamp_ttl'] = int(
(datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1)).total_seconds() +
master_ttl)
update_dyanmo_summary(rekog_summary_dynamo)
## Found Label, send over to PREKOG lambda
if rekog_summary_dynamo['rekog_label'] == REKOG_LABEL:
dynamo_object['rekog_label'] = obj['Name']
invoke_lambda(dynamo_object)
# if int(round(obj['Confidence'])) > 49:
update_dyanmo_summary(rekog_summary_dynamo)
if obj['Name'] == 'Text':
text_ok = 1
if obj['Name'] == 'Person':
person_ok = 1
rekog_labels = sorted(rekog_labels_list,
key=lambda obj: obj['Confidence'])
print('rekog labels: {}'.format(rekog_labels))
dynamo_object['rekog_labels'] = rekog_labels
## REKOGNTION -- Text
if text_ok == 1:
print("rekog found Text, running OCR....")
rekog_word_list = []
rekog_text = rekognition.detect_text(
Image={
"S3Object": {
"Bucket": S3_BUCKET,
"Name": 'images/' + dynamo_filename
}
})
print('rekog words: {}'.format(rekog_text))
if rekog_text['TextDetections']:
print('rekog found text in the image')
for obj in rekog_text['TextDetections']:
if obj['Type'] == 'WORD':
rekog_word_list.append({
'Confidence':
str(int(round(obj['Confidence']))),
'Name':
obj['DetectedText']
})
## deltafa-summary
rekog_summary_dynamo = {}
rekog_summary_dynamo['scenedetect'] = str(
scenedetect[str(x_filename)])
rekog_summary_dynamo['rekog_label'] = obj[
'DetectedText']
rekog_summary_dynamo['rekog_type'] = 'word'
rekog_summary_dynamo[
'id_filename'] = dynamo_filename
rekog_summary_dynamo[
'timestamp_updated'] = int(
(datetime.datetime.utcnow() -
datetime.datetime(1970, 1,
1)).total_seconds())
rekog_summary_dynamo['timestamp_ttl'] = int(
(datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1)
).total_seconds() + master_ttl)
update_dyanmo_summary(rekog_summary_dynamo)
## Found Label, send over to PREKOG lambda
if rekog_summary_dynamo['rekog_label'].upper(
) == REKOG_LABEL.upper():
dynamo_object['rekog_label'] = obj['Name']
invoke_lambda(dynamo_object)
print('rekog words: {}'.format(rekog_word_list))
dynamo_object['rekog_words'] = rekog_word_list
else:
print('rekog did NOT find words...')
## REKOGNTION -- Celeb
if person_ok == 1:
print("rekog found a Person, running celeb....")
rekog_celebs = rekognition.recognize_celebrities(
Image={
"S3Object": {
"Bucket": S3_BUCKET,
"Name": 'images/' + dynamo_filename
}
})
if rekog_celebs['CelebrityFaces']:
print("rekog found a celeb...")
rekog_celebs_list = []
for obj in rekog_celebs['CelebrityFaces']:
# invoke_lambda( rekog_label_dynamo )
## deltafa-summary
rekog_summary_dynamo = {}
rekog_summary_dynamo['scenedetect'] = str(
scenedetect[str(x_filename)])
rekog_summary_dynamo['rekog_label'] = obj['Name']
rekog_summary_dynamo['rekog_type'] = 'celeb'
rekog_summary_dynamo[
'id_filename'] = dynamo_filename
rekog_summary_dynamo['timestamp_updated'] = int((
datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1)).total_seconds())
rekog_summary_dynamo['timestamp_ttl'] = int(
(datetime.datetime.utcnow() -
datetime.datetime(1970, 1, 1)
).total_seconds() + master_ttl)
update_dyanmo_summary(rekog_summary_dynamo)
print('rekgo celeb: {}'.format(rekog_celebs_list))
dynamo_object['rekog_celebs'] = rekog_celebs_list
else:
print("rekog did NOT find celeb...")
# PUT DYNAMO - main
put_dynamo_main(dynamo_object)
scenechange_list.append(dynamo_object['id_filename'])
# clean up jpgs
delete_file(tmpdir + dynamo_filename)
x_frame += 1
x_filename += 1
# full segment information
dynamo_segment_object['scenechange_list'] = scenechange_list
pprint(dynamo_segment_object)
put_dynamo_main(dynamo_segment_object)
write_json()
return 'SUCCESS: it ran'
def enable_xray():
patch_all()
def create_app():
app = Flask(__name__)
# Configure xray tracing
xray_recorder.configure(service='dashboard')
XRayMiddleware(app, xray_recorder)
patch_all()
# Setup logging
setup_logging(log_levels.get(os.environ.get("LOG_LEVEL", "INFO")))
log = logging.getLogger(__name__)
@app.errorhandler(HTTPException)
def handle_error(error):
error_dict = {
"message": str(error) if isinstance(error, HTTPException) else '',
"exc_type": type(error).__name__,
"status_code": error.code
}
log.exception(error)
return make_response(
json.dumps({
"message":
"Error occured",
"code":
error.code,
"trace_id":
get_trace_id(request.headers.get("X-Amzn-Trace-Id")),
"request_id":
get_request_id()
}), 500)
@app.route("/")
def hello():
try:
resp = requests.get("{}/".format(COUNTER_ENDPOINT),
headers={"X-Request-Id": get_request_id()})
log.info("Received request, trace_id={}, req_id={}".format(
get_trace_id(request.headers.get("X-Amzn-Trace-Id")),
get_request_id()))
if resp.status_code == 200:
resp = json.loads(resp.text)
resp = json.dumps({
"message":
"Counter is reachable",
"count":
resp["count"],
"counter_service_id":
resp["counter_service_id"],
"dashboard_service_id":
socket.gethostname(),
"trace_id":
get_trace_id(request.headers.get("X-Amzn-Trace-Id")),
"request_id":
get_request_id()
})
else:
log.info(
"Error calling {} service! code={}, service={}".format(
resp.text, resp.status_code, COUNTER_ENDPOINT))
resp = make_response(
json.dumps({
"message":
"Error calling %s service" % COUNTER_ENDPOINT,
"code":
resp.status_code,
"error":
resp.text,
"trace_id":
get_trace_id(request.headers.get("X-Amzn-Trace-Id")),
"request_id":
get_request_id()
}), 500)
except requests.exceptions.RequestException as ex:
log.info("Error connecting to counter at %s, %s" %
(COUNTER_ENDPOINT, str(ex)))
resp = make_response(
json.dumps({
"message":
str(ex),
"trace_id":
get_trace_id(request.headers.get("X-Amzn-Trace-Id")),
"request_id":
get_request_id()
}), 500)
return resp
@app.route("/fail")
def fail():
code = request.args.get('code')
log.info(
"/fail?code={} called! Relaying to counter service, trace_id={}, req_id={}"
.format(code, get_trace_id(request.headers.get("X-Amzn-Trace-Id")),
get_request_id()))
resp = requests.get("{}/fail?code={}".format(COUNTER_ENDPOINT, code),
headers={"X-Request-Id": get_request_id()})
resp = make_response(
json.dumps({
"message":
"Received code={} on {}/fail?code={}".format(
resp.status_code, COUNTER_ENDPOINT, code)
}), int(resp.status_code))
return resp
@app.route("/health")
def health():
return json.dumps({"status": "DASHBOARD_HEALTHY"})
return app
# required for serverless framework to un-compress binaries
import json
import os
import sys
import logging
import re
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
import boto3
from aws_xray_sdk.core import xray_recorder
from aws_xray_sdk.core import patch_all
patch_all(
) # This will patch boto3 and any other supported client library to emit xray data.
# Credentails for the lambdas are set via IAM roles in the serverless.yml file.
lambda_client = boto3.client("lambda") # client object to access lambda APIs
sqs_client = boto3.client("sqs")
# Let's figure out what stage we are running and set the logging and xray configuration based on it.
DEPLOYMENT_STAGE = os.environ.get("DEPLOYMENT_STAGE")
if DEPLOYMENT_STAGE is None:
logger.debug("No DEPLOYMENT_STAGE environment variable set. Assuming dev.")
DEPLOYMENT_STAGE = (
"dev" # The default stage will be dev even if nothing is specified.
)
# Next I want to set a specific xray sampling configurarion.
def lambda_handler(event, context):
if XRAY == 'true':
patch_all()
get_environment_variables()
timestamp_start = (datetime.datetime.now() - datetime.timedelta(seconds=60))
timestamp_end = datetime.datetime.now()
timestamps = {
'start' : "{:%b %d, %H:%M:%S}".format(timestamp_start),
'end' : "{:%b %d, %H:%M:%S}".format(timestamp_end),
}
print('timestamps: {}'.format(timestamps))
full_list = []
images_list = {}
rekog_type = 'label'
this_response = get_dynamo_summary(rekog_type)
for stuff in this_response:
full_list.append(stuff['rekog_label'].lower())
# images_list.append({stuff['rekog_label'].lower() : stuff['id_filename']})
images_list[stuff['rekog_label'].lower()] = stuff['id_filename']
rekog_type = 'word'
this_response = get_dynamo_summary(rekog_type)
for stuff in this_response:
full_list.append(stuff['rekog_label'].lower())
# images_list.append({stuff['rekog_label'].lower() : stuff['id_filename']})
images_list[stuff['rekog_label'].lower()] = stuff['id_filename']
print('full list: {}'.format(full_list))
print('images list: {}'.format(images_list))
# boulder eats: [u'girl', u'salad', u'human', u'woman', u'blonde', u'food', u'dinner', u'face', u'bowl', u'supper', u'lunch', u'female', u'person', u'meal', u'people', u'portrait', u'sport', u'sports', u'working out', u'produce', u'exercise', u'long sleeve', u'dahlia', u'sprout', u'daisy', u'clothing', u'flora', u'ornament', u'daisies', u'flower']
# top gear: [u'human', u'vehicle', u'transportation', u'face', u'automobile', u'driving', u'person', u'people', u'portrait', u'car', u'convertible', u'apparel', u'shoe', u'clothing', u'footwear', u'wedge', u'triangle', u'laptop', u'pc', u'computer', u'electronics', u'bumper', u'white board', u'mold', u'adorable', u'sink', u'coupe', u'sports car', u'arm', u'airport']
# top gear 2: [u'human', u'vehicle', u'transportation', u'face', u'automobile', u'driving', u'person', u'people', u'portrait', u'car', u'convertible', u'apparel', u'shoe', u'clothing', u'footwear', u'wedge', u'triangle', u'laptop', u'pc', u'computer', u'electronics', u'bumper', u'white board', u'mold', u'adorable', u'sink', u'coupe', u'sports car', u'arm', u'airport']
# zoo bear: [u'mammal', u'hole', u'bear', u'wildlife', u'animal', u'zoo', u'astronomy', u'space', u'universe', u'outer space']
# hockey: [u'skating', u'human', u'team sport', u'hockey', u'sport', u'ice hockey', u'team', u'ice skating', u'sports', u'person', u'rink', u'people', u'modern art', u'art']
match_list = {
'formula_racing' : ['road', 'street', 'vehicle', 'wheel' ],
'moods_tour' : ['instrument', 'band', 'guitar', 'city', 'alcohol'],
'odeur' : ['clothing', 'flora'],
'robots' : ['toys', 'electronics'],
'save_polarbear' : ['animal', 'bear', 'cat', 'dog'],
'sportz' : ['baseball', 'football', 'basketball', 'hockey', 'sports'],
'visit_alps' : ['ski', 'snow', 'powder', 'luggage',],
'visit_caribbean' : ['beach', 'sun', 'drinks', 'nature', 'weather', 'sunset' ],
'g_i_joe' : ['toy', 'sport', 'outdoors'],
'candy_bar' : ['salad'],
'video_game' : ['watch', 'television', 'electronics'],
'super_boy' : ['grass', 'outer space', ],
'toy_puppy' : ['toy', 'dog', 'zoo', 'bird'],
'action_figure' : ['toy', 'gun', 'motorcycle', ],
# 'toy_car' : ['toy', 'car', 'electronics'],
'chocoloate_drink' : ['drinks', 'food', 'lunch'],
# 'soft_drink' : ['drinks', 'glass', 'beverage'],
}
thirty_secs = [
'g_i_joe',
'candy_bar',
'video_game',
'super_boy',
'toy_puppy',
'action_figure',
'toy_car',
'chocoloate_drink',
'soft_drink',
]
ad_playlist = []
ad_matches = []
images_matches = {}
for ad_type in match_list:
for ad_label in match_list[ad_type]:
if ad_label in full_list:
print('MATCHED! type: {} \tlabel: {}'.format(ad_type,ad_label))
ad_playlist.append(ad_type)
ad_matches.append({ad_type : ad_label})
images_matches[ad_label] = {'image' : images_list[ad_label], 'type': ad_type }
print('images_matches: {}'.format(images_matches))
# pprint(ad_playlist)
if len(ad_playlist) < 1:
ad_playlist.append('default')
while len(ad_playlist) < 5:
ad_playlist.extend(ad_playlist)
print('playlist lineup: {}'.format(ad_playlist))
full_xml = '''<VAST version="3.0">
'''
x = 0
for ad in ad_playlist:
duration = '00:00:15'
if ad in thirty_secs:
duration = '00:00:30'
x += 1
full_xml += '''
<Ad sequence="''' + str(x) + '''">
<InLine>
<AdSystem>2.0</AdSystem>
<AdTitle>ad-''' + str(x) + '''</AdTitle>
<Impression></Impression>
<Creatives>
<Creative id="''' + str(ad) + '''-3">
<Linear>
<Duration>''' + str(duration) + '''</Duration>
<TrackingEvents></TrackingEvents>
<MediaFiles>
<MediaFile delivery="progressive" type="video/mp4" width="1920" height="1080">
<![CDATA[
https://s3.amazonaws.com/ces2018-demo/ads/''' + ad + '''-3.mp4
]]>
</MediaFile>
</MediaFiles>
</Linear>
</Creative>
</Creatives>
</InLine>
</Ad> '''
full_xml += '''
</VAST>'''
# print('xml: {}'.format(full_xml))
json_together = {
# 'full_list' : full_list,
'matches' : ad_matches,
'images_list' : images_list,
'images_matches' : images_matches,
'timestamps' : timestamps,
}
# pprint(json_together)
json_string = json.dumps(json_together, cls=DecimalEncoder)
dynamo_filename = 'list-ads.json'
## set tmp directory
tmpdir = '/tmp/' + str(uuid.uuid4()) + '/'
ensure_dir(tmpdir)
with open(tmpdir + dynamo_filename, 'w') as outfile:
outfile.write(json_string)
## S3 upload
data = open(tmpdir + dynamo_filename, 'rb')
s3 = boto3.resource('s3')
pprint(s3.Bucket(S3_BUCKET).put_object(Key=dynamo_filename, Body=data))
return {
'statusCode': '200',
# 'body': re.sub(r'(.+m3u8|.+ts)', base_url + r'\1', master,re.M),
'body': full_xml,
'headers': {'Content-Type': 'application/xml', 'Access-Control-Allow-Origin': '*'}
}
def lambda_handler(event, context):
if XRAY == 'true':
patch_all()
### SNS from MediaConvert
if 'detail-type' in event:
if event['detail-type'] == 'MediaConvert Job State Change':
emc_list = {
'label' : event['detail']['userMetadata']['label'],
'label_sort' : event['detail']['userMetadata']['label_sort'],
'emc_url' : event['detail']['userMetadata']['label_sort'] + '_video.mp4' ,
'emc_status' : event['detail']['status'],
}
update_dyanmo_list ( emc_list )
else:
# set here because of global variable
mediaconvert = boto3.client('mediaconvert', endpoint_url=EMC_ENDPOINT)
master_ttl = 3600
url_full = event['emp_url']
s3_bucket = event['s3_bucket']
event_id = event['label_sort']
# PARSE manifest
print("master manifest: " + url_full)
# wrangle the urls
base_url = '/'.join(url_full.split('/')[:-1]) + '/'
print("baseurl: {}".format(base_url))
filename_master = url_full.split('/')[-1]
print("filename_master: {}".format(filename_master))
# GET master manifest
string_master = get_url(base_url + filename_master)
print("string_master: {}".format(string_master))
# PARSE the m3u8 child manifestss. Returns list of these strings
filename_playlists = [x for x in string_master.split('\n') if '.m3u8' in x]
print("filename_playlists: {}".format(filename_playlists))
# GET child manifest
string_playlist = get_url(base_url + filename_playlists[0])
print("string_playlist: {}".format(string_playlist))
# get filenames assumption started
# filename_base = segment_filename.split('.ts')[0]
# print('filename_base: {}'.format(filename_base))
## set tmp directory
tmpdir = '/tmp/' + event_id + '/'
ensure_dir(tmpdir)
print('tmpdir: {}'.format(tmpdir))
# get the physical master manifest file
save_file(tmpdir + filename_master.split('?')[0], get_url(base_url + filename_master))
data = open(tmpdir + filename_master.split('?')[0], 'rb')
pprint(s3.Bucket(s3_bucket).put_object(Key='video' + tmpdir + filename_master.split('?')[0], Body=data))
delete_file(tmpdir + filename_master.split('?')[0])
# get the physical child manifest file
save_file(tmpdir + filename_playlists[0].split('?')[0], get_url(base_url + filename_playlists[0]))
data = open(tmpdir + filename_playlists[0].split('?')[0], 'rb')
pprint(s3.Bucket(s3_bucket).put_object(Key='video' + tmpdir + filename_playlists[0].split('?')[0], Body=data))
delete_file(tmpdir + filename_playlists[0].split('?')[0])
# get the physical segment file
for x in string_playlist.split('\n'):
if '.ts' in x:
pprint(x)
segment_filename = x.split('?')[0]
save_file(tmpdir + segment_filename, get_url(base_url + segment_filename))
data = open(tmpdir + segment_filename, 'rb')
pprint(s3.Bucket(s3_bucket).put_object(Key='video' + tmpdir + segment_filename, Body=data))
delete_file(tmpdir + segment_filename)
## MediaConvert
# mediaConvertRole = "arn:aws:iam::1234567890:role/vod-MediaConvertIAMRole-1BIVDOODMZT1L-MediaConvertRole"
# mediaConvertEndpoint = "https://vweb51sm.mediaconvert.us-west-2.amazonaws.com"
time_start = str(event['time_start'].split(' ')[1]) + ":" + event['time_start_frame'].zfill(2)
time_end = str(event['time_end'].split(' ')[1]) + ":" + event['time_end_frame'].zfill(2)
# time_start = event['time_start'].split(' ')[1]
# time_end = event['time_end'].split(' ')[1]
# mediaconvert = boto3.client('mediaconvert', endpoint_url=EMC_ENDPOINT)
jobMetadata = {'label_sort': str(event['label_sort']), 'label': str(event['label']) }
jobSettings = {
"OutputGroups": [
{
"Name": "File Group",
"Outputs": [
{
"ContainerSettings": {
"Container": "MP4",
"Mp4Settings": {
"CslgAtom": "INCLUDE",
"FreeSpaceBox": "EXCLUDE",
"MoovPlacement": "PROGRESSIVE_DOWNLOAD"
}
},
"VideoDescription": {
"Width": 1280,
"ScalingBehavior": "DEFAULT",
"Height": 720,
"TimecodeInsertion": "DISABLED",
"AntiAlias": "ENABLED",
"Sharpness": 50,
"CodecSettings": {
"Codec": "H_264",
"H264Settings": {
"InterlaceMode": "PROGRESSIVE",
"NumberReferenceFrames": 3,
"Syntax": "DEFAULT",
"Softness": 0,
"GopClosedCadence": 1,
"GopSize": 90,
"Slices": 1,
"GopBReference": "DISABLED",
"SlowPal": "DISABLED",
"SpatialAdaptiveQuantization": "ENABLED",
"TemporalAdaptiveQuantization": "ENABLED",
"FlickerAdaptiveQuantization": "DISABLED",
"EntropyEncoding": "CABAC",
"Bitrate": 5000000,
"FramerateControl": "INITIALIZE_FROM_SOURCE",
"RateControlMode": "CBR",
"CodecProfile": "MAIN",
"Telecine": "NONE",
"MinIInterval": 0,
"AdaptiveQuantization": "HIGH",
"CodecLevel": "AUTO",
"FieldEncoding": "PAFF",
"SceneChangeDetect": "ENABLED",
"QualityTuningLevel": "SINGLE_PASS",
"FramerateConversionAlgorithm": "DUPLICATE_DROP",
"UnregisteredSeiTimecode": "DISABLED",
"GopSizeUnits": "FRAMES",
"ParControl": "INITIALIZE_FROM_SOURCE",
"NumberBFramesBetweenReferenceFrames": 2,
"RepeatPps": "DISABLED"
}
},
"AfdSignaling": "NONE",
"DropFrameTimecode": "ENABLED",
"RespondToAfd": "NONE",
"ColorMetadata": "INSERT"
},
"AudioDescriptions": [
{
"AudioTypeControl": "FOLLOW_INPUT",
"CodecSettings": {
"Codec": "AAC",
"AacSettings": {
"AudioDescriptionBroadcasterMix": "NORMAL",
"Bitrate": 96000,
"RateControlMode": "CBR",
"CodecProfile": "LC",
"CodingMode": "CODING_MODE_2_0",
"RawFormat": "NONE",
"SampleRate": 48000,
"Specification": "MPEG4"
}
},
"LanguageCodeControl": "FOLLOW_INPUT"
}
],
"NameModifier": "_video"
}
],
"OutputGroupSettings": {
"Type": "FILE_GROUP_SETTINGS",
"FileGroupSettings": {
"Destination": "s3://" + s3_bucket + "/video/" + event_id
}
}
}
],
"AdAvailOffset": 0,
"Inputs": [
{
"InputClippings": [
{
"EndTimecode": time_end,
"StartTimecode": time_start
}
],
"AudioSelectors": {
"Audio Selector 1": {
"Offset": 0,
"DefaultSelection": "DEFAULT",
"ProgramSelection": 1
}
},
"VideoSelector": {
"ColorSpace": "FOLLOW"
},
"FilterEnable": "AUTO",
"PsiControl": "USE_PSI",
"FilterStrength": 0,
"DeblockFilter": "DISABLED",
"DenoiseFilter": "DISABLED",
"TimecodeSource": "EMBEDDED",
"FileInput": "s3://" + s3_bucket + "/video/tmp/" + event_id + "/index.m3u8"
}
]
}
pprint(jobSettings)
job = mediaconvert.create_job(Role=EMC_ROLE, UserMetadata=jobMetadata, Settings=jobSettings)
# pprint(job)
return 'SUCCESS: it ran'
BOTO_RESOURCE_KWARGS={"region_name": aws_region},
CELL_SETS_BUCKET=f"cell-sets-{cluster_env}",
SOURCE_BUCKET=f"processed-matrix-{cluster_env}",
RESULTS_BUCKET=f"worker-results-{cluster_env}",
R_WORKER_URL="http://localhost:4000",
# this works because in CI, `data/` is deployed under `worker/`
# whereas in a container, it is mounted to `/data`. Either way, this ensures
# that the appropriate path is selected, as both are two directories up
LOCAL_DIR=os.path.join(os.pardir, os.pardir, "data"),
)
config.API_URL = (
f"http://api-{config.SANDBOX_ID}.api-{config.SANDBOX_ID}.svc.cluster.local:3000"
)
if cluster_env == "development" or cluster_env == "test":
config.AWS_ACCOUNT_ID = "000000000000"
config.BOTO_RESOURCE_KWARGS["aws_access_key_id"] = "my-key"
config.BOTO_RESOURCE_KWARGS["aws_secret_access_key"] = "my-secret-key"
config.API_URL = "http://host.docker.internal:3000"
global_sdk_config.set_sdk_enabled(False)
if cluster_env == "development":
config.BOTO_RESOURCE_KWARGS[
"endpoint_url"] = "http://host.docker.internal:4566"
config.R_WORKER_URL = "http://r:4000"
if cluster_env != "test":
core.patch_all()
def lambda_handler(event, context):
if XRAY == 'true':
patch_all()
get_environment_variables()
print('event: {}'.format(event))
print('S3_BUCKET: {}'.format(S3_BUCKET))
print('EMP_URL: {}'.format(EMP_URL))
if EMP_URL == 'not-set':
return 'ERROR: EMP_URL was not set'
rekog_label = event['rekog_label']
id_filename = event['id_filename']
print('rekog_label: {} \tid_filename: {}'.format(rekog_label, id_filename))
## BACKWARDS in time
if int(event['scenedetect']) > 50:
print('first scene is {}, skipping hops_backwards'.format(
event['scenedetect']))
hops_backward = {
'last_hop':
event,
'messages': [
'first scene is {}, skipping hops_backwards'.format(
event['scenedetect'])
],
'all_hops': {
'1': event
},
}
else:
hops_backward = detect_hops(rekog_label, id_filename, False)
print('hops_backward: {}'.format(hops_backward))
## FORWARD in time ( may have to wait for time to actually happen )
hops_forward = detect_hops(rekog_label, id_filename, True)
print('hops_forward: {}'.format(hops_forward))
query_dict = {}
query_dict['label_image'] = id_filename
query_dict['label'] = rekog_label
query_dict['time_start'] = hops_backward['last_hop'][
'timestamp_minute'] + ':' + hops_backward['last_hop'][
'timestamp_second']
query_dict['time_start_frame'] = str(
hops_backward['last_hop']['timestamp_frame'])
query_dict['time_start_image'] = hops_backward['last_hop']['id_filename']
query_dict['messages_backward'] = hops_backward['messages']
query_dict['hops_backward'] = hops_backward['all_hops']
query_dict['time_end'] = hops_forward['last_hop'][
'timestamp_minute'] + ':' + hops_forward['last_hop']['timestamp_second']
query_dict['time_end_frame'] = str(
hops_forward['last_hop']['timestamp_frame'])
query_dict['time_end_image'] = hops_forward['last_hop']['id_filename']
query_dict['messages_forward'] = hops_forward['messages']
query_dict['hops_forward'] = hops_forward['all_hops']
# pprint(query_dict)
query_dict[
'emp_url'] = EMP_URL + '?start=' + query_dict['time_start'].replace(
' ', 'T') + '+00:00&end=' + query_dict['time_end'].replace(
' ', 'T') + '+00:00'
lambda_event = put_dynamo(query_dict)
pprint(lambda_event)
invoke_lambda(lambda_event)
return 'SUCCESS: it ran'
