def main(args):
ARN = "arn:aws:states:us-east-1:497940546915:activity:transcodeVideo"
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SFN boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
sfn = boto3.client('stepfunctions', config=botoConfig)
logging.debug("Created SFN boto client: %s", sfn)
while True:
task = sfn.get_activity_task(activityArn=ARN,
workerName='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['ResponseMetadata']['RequestId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
# Take the thumbnail 25% through the video
newDir = "converted"
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
# We require the %d to keep the file names incremented
# Note that we need to escape the percentage sign by using another %, hence the double %
outfile = '%s_PDL.mp4' % (fileName)
cmd = [
'ffmpeg', '-y', '-i', asset, '-c:v', 'libx264', '-preset',
'fast', '-crf', '22', '-c:a', 'aac', '-loglevel', 'fatal',
'%s/%s' % (subDir, outfile)
]
logging.debug("[%s] Execute video transcoding PDL creation: %s",
workID, cmd)
try:
output = subprocess.check_output(cmd)
OUTPUT = {
'tool': output,
'dbPrimaryKey': dbPrimaryKey,
'assetClass': INPUT['assetClass'],
'asset': asset,
}
logging.debug("[%s] Update PDL value", workID)
updateExpression = 'set PDL = :t'
expressionValues = {
':t': '/%s/%s' % (newDir, outfile),
}
response = databaseHelper.updateEntry(dbPrimaryKey,
updateExpression,
expressionValues)
sfn.send_task_success(taskToken=taskToken,
output=json.dumps(OUTPUT))
# We should catch other errors here
except subprocess.CalledProcessError as err:
logger.log('E', err.returncode)
result = {
'reason': 'TRC-0001_Error in MP4 conversation',
'detail': str(err)
}
logging.error("%s", result)
sfn.send_task_failure(taskToken=taskToken,
error=json.dumps(result['reason']),
cause=json.dumps(result['detail']))
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
DOMAIN = 'ITD'
TASKLIST = 'default'
VERSION = '1'
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SWF boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
swf = boto3.client('swf', config=botoConfig)
logging.debug("Created SWF boto client: %s", swf)
while True:
task = swf.poll_for_activity_task(domain=DOMAIN,
taskList={'name': taskName},
identity='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['workflowExecution']['workflowId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
# Take the thumbnail 25% through the video
scale = "640x360"
newDir = "thumbnails"
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
outfile = '%s_thumbnail.jpg' % (fileName)
cmd = [
'convert', asset, '-format', 'jpg', '-thumbnail', scale,
'-auto-orient',
'%s/%s' % (subDir, outfile)
]
logging.debug("[%s] Execute image thumbnail creation: %s", workID,
cmd)
try:
output = subprocess.check_output(cmd)
# Start setting the parameters needed to update the thumbnail
key = dbPrimaryKey
updateExpression = 'set thumbnail = :t'
expressionValues = {':t': '/%s/%s' % (newDir, outfile)}
logging.debug("[%s] Update thumbnail value", workID)
# Call the update function
response = databaseHelper.updateEntry(key, updateExpression,
expressionValues)
OUTPUT = {
'tool': output,
'dbPrimaryKey': dbPrimaryKey,
'assetClass': INPUT['assetClass'],
'asset': asset,
}
swf.respond_activity_task_completed(taskToken=taskToken,
result=json.dumps(OUTPUT))
# We should catch other errors here
except subprocess.CalledProcessError as err:
result = {
'reason': 'THB-0001_Error in image thumbnail creation',
'detail': str(err)
}
logging.error("%s", result)
swf.respond_activity_task_failed(
taskToken=taskToken,
reason=json.dumps(result['reason']),
details=json.dumps(result['detail']))
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
DOMAIN = 'ITD'
TASKLIST = 'default'
VERSION = '1'
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SWF boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
swf = boto3.client('swf', config=botoConfig)
logging.debug("Created SWF boto client: %s", swf)
while True:
task = swf.poll_for_activity_task(domain=DOMAIN,
taskList={'name': taskName},
identity='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['workflowExecution']['workflowId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
# Take the thumbnail 25% through the video
#scale = "640x360"
# Use the multipliers so that we don't distort vertical videos. This makes it generic.
scale = "iw/3:ih/3" # 1/3 gives 1920 (HD) down to 640
fps = 1 # Set the number of frames to be once per second
newDir = "thumbnails"
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
# We require the %d to keep the file names incremented
# Note that we need to escape the percentage sign by using another %, hence the double %
outfile = '%s_thumbnail_%%d.jpg' % (fileName)
vtt = '%s.vtt' % (fileName)
# Parameters are
# -y for
# -i for Input
# -vf, fps=1,scale= for the video filter stating we want to take every one second
cmd = [
'ffmpeg', '-y', '-i', asset, '-vf',
'fps=%s,scale=%s' % (fps, scale), '-loglevel', 'fatal',
'%s/%s' % (subDir, outfile)
]
logging.debug("[%s] Execute video thumbnail creation: %s", workID,
cmd)
try:
output = subprocess.check_output(cmd)
# Start setting the parameters needed to update the thumbnail
# Comment block is staying for reference sake
'''# Call the update function
# The "thumbnails" map will need to be created if it doesn't exist (Note: It shouldn't at this point)
# A validation exception will be thrown, and when this is thrown, we will create an empty map and try it again
try:
response = databaseHelper.updateEntry(key, updateExpression, expressionValues)
except botocore.exceptions.ClientError as err:
if err.response['Error']['Code'] == 'ValidationException':
response = databaseHelper.updateEntry(key, 'set thumbnails = :t', {':t' : {}})
response = databaseHelper.updateEntry(key, updateExpression, expressionValues)
'''
# After the thumbnails are created, we need to do two things:
# OLD # 1. Create the storyboard object which is [http://docs.brightcove.com/en/perform/brightcove-player/guides/thumbnails-plugin.html#collectimages]
# 1. Create the storyboard VTT file (https://support.jwplayer.com/customer/portal/articles/1407439-adding-preview-thumbnails)
# 2. We also need to identify the thumbnail for the video which we will take a percentage of the way through the video
#STORYBOARD = {}
thumbnailTime = .25 # Pick the thumbnail that's 25% of the way through the video
counter = 0
for thumb in os.listdir(subDir):
if fnmatch.fnmatch(
thumb, '*_thumbnail_*.jpg'
): # Match files in the directory that are the thumbnails
#sequenceNum = thumb[thumb.rfind('_')+1:-4] # filename_thumbnail_$frame.jpg
#STORYBOARD[sequenceNum] = {'src' : '/%s/%s' %(newDir, thumb) }
counter = counter + 1
# Open the VTT file and write
logging.debug("[%s] Writing VTT file: %s", workID, vtt)
vttFile = open('%s/%s' % (subDir, vtt), 'w')
vttFile.write("WEBVTT")
# The counter represents how many files of FPS we have -- range is COUNTER*FPS --> (COUNTER+1)* fps
# FPS references the frames per second so if we put (1/60), that means a frame EVERY MINUTE
# Therefore, we need to invest the FPS
# Use %02d to PAD the numbers
baseURL = "https://dnt4vq51jg2tj.cloudfront.net" # There needs to be a better way then the full URL
for i in range(0, counter):
startSecond = i * (1 / fps)
endSecond = (i + 1) * (1 / fps)
startSpan = '%02d:%02d:%02d.000' % (startSecond / 3600,
startSecond / 60 % 60,
startSecond % 60)
endSpan = '%02d:%02d:%02d.000' % (
endSecond / 3600, endSecond / 60 % 60, endSecond % 60)
thumbSpan = '%s/%s/%s/%s_thumbnail_%d.jpg' % (
baseURL, fileName, newDir, fileName, i + 1)
vttFile.write("\n\n%s --> %s\n%s" %
(startSpan, endSpan, thumbSpan))
vttFile.close()
logging.debug("[%s] Wrote VTT file: %s", workID, vtt)
index = str(math.trunc(counter * thumbnailTime))
logging.debug("[%s] Key frame identified in index: %s", workID,
index)
updateExpression = 'set thumbnail = :t, storyboard = :s'
thumbnail = '/%s/%s_thumbnail_%s.jpg' % (newDir, fileName,
index)
# THERE MUST BE A DYNAMIC WAY TO DO THIS BUT I DONT KNOW YET
storyboard = '/%s/%s' % (newDir, vtt)
'''expressionValues = {
':t' : STORYBOARD[index]['src'],
':s' : STORYBOARD
}'''
expressionValues = {
':t': thumbnail,
':s': storyboard,
}
logging.debug("[%s] Update thumbnail value", workID)
response = databaseHelper.updateEntry(dbPrimaryKey,
updateExpression,
expressionValues)
OUTPUT = {
'tool': output,
'dbPrimaryKey': dbPrimaryKey,
'assetClass': INPUT['assetClass'],
'asset': asset,
}
swf.respond_activity_task_completed(taskToken=taskToken,
result=json.dumps(OUTPUT))
# We should catch other errors here
except subprocess.CalledProcessError as err:
result = {
'reason': 'THB-0002_Error in video thumbnail creation',
'detail': str(err)
}
logging.error("%s", result)
swf.respond_activity_task_failed(
taskToken=taskToken,
reason=json.dumps(result['reason']),
details=json.dumps(result['detail']))
logging.info("[%s] %s Complete", workID, taskName)
def transcodeTest():
asset = '/Assets/test/trim.6A7FE5FA-5321-4BF1-87D7-D26937142263_76ed9582-7da6-44ef-a0af-e22596297573/trim.6A7FE5FA-5321-4BF1-87D7-D26937142263_76ed9582-7da6-44ef-a0af-e22596297573.MOV'
scale = "640x360"
newDir = "thumbnails"
thumbnailTime = .25 # Create thumbnail 25% of the way through the video
fps = 1
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
# We require the %d to keep the file names incremented
# Note that we need to escape the percentage sign by using another %, hence the double %
outfile = '%s_thumbnail_%%d.jpg' % (fileName)
vtt = '%s.vtt' % (fileName)
cmd = ['ffmpeg'
,'-y'
,'-i', asset
,'-s', scale
,'-vf', 'fps=%s' % (fps)
,'%s/%s' %(subDir, outfile)
]
output = subprocess.check_output(cmd)
print output
# After the thumbnails are created, we need to do two things:
# 1. Find the thumbnail we want to display, and create the object
# 2. Create the storyboard object which is [http://docs.brightcove.com/en/perform/brightcove-player/guides/thumbnails-plugin.html#collectimages]
# "second" : { "src" : "thumbnail"}
STORYBOARD = {}
counter = 0
for thumb in os.listdir(subDir):
if fnmatch.fnmatch(thumb, '*_thumbnail_*.jpg'):
counter = counter + 1
# Open the VTT file and write
vttFile = open('%s/%s' %(subDir, vtt), 'w')
vttFile.write("WEBVTT")
# The counter represents how many files of FPS we have -- range is COUNTER*FPS --> (COUNTER+1)* fps
# FPS references the frames per second so if we put (1/60), that means a frame EVERY MINUTE
# Therefore, we need to invest the FPS
# Use %02d to PAD the numbers
for i in range(0,counter):
startSecond = i * (1/fps)
endSecond = (i + 1) * (1/fps)
startSpan = '%02d:%02d:%02d.000' % ( startSecond / 3600, startSecond / 60 % 60, startSecond % 60)
endSpan = '%02d:%02d:%02d.000' % ( endSecond / 3600, endSecond / 60 % 60, endSecond % 60)
thumbSpan = '%s/%s_thumbnail_%d.jpg' % (newDir, fileName,i + 1)
vttFile.write("\n\n%s --> %s\n%s" % (startSpan, endSpan, thumbSpan))
vttFile.close()
index = str(math.trunc(counter * thumbnailTime))
thumbnail = '%s/%s_thumbnail_%s.jpg' % (newDir, fileName,index)
sha1 = '6e508f0ed03da2b98ceb091827b569877037d1ef'
# Start setting the parameters needed to update the thumbnail
key = {
'Checksum' : sha1,
}
updateExpression = 'set thumbnail = :t, storyboard = :s'
expressionValues = {
':t' : thumbnail,
':s' : '%s/%s' %(newDir, vtt)
}
def main(args):
DOMAIN = 'ITD'
VERSION = '1'
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SWF boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
swf = boto3.client('swf', config=botoConfig)
logging.debug("Created SWF boto client: %s", swf)
BUCKETNAME = "schulerfiles"
workingStorage = "/Assets/working/"
while True:
task = swf.poll_for_activity_task(domain=DOMAIN,
taskList={'name': taskName},
identity='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['workflowExecution']['workflowId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
source = INPUT['locationSource']
destination = INPUT['locationDestination']
dbPrimaryKey = INPUT['dbPrimaryKey']
fileKey = INPUT['fileKey'] + '/'
# Bucket object is necessary in all cases
logging.debug("[%s] Creating S3 bucket boto client", workID)
s3 = boto3.resource('s3')
bucket = s3.Bucket(BUCKETNAME)
logging.debug("[%s] Created S3 bucket boto client", workID)
# Setting the storage class to be used for later
s3StorageClass = 'STANDARD'
if destination == 'near_line':
s3StorageClass = 'STANDARD_IA'
logging.info("[%s] Moving %s from %s to %s", workID, fileKey,
source, destination)
# CDN and near_line are both S3 tiers, so all we are doing is changing the Storage Class with a PUT
if (source == 'CDN' and destination == 'near_line') or (
source == 'near_line' and destination == 'CDN'):
logging.info("[%s] Moving objects between S3 and S3IA", workID)
for obj in bucket.objects.filter(Prefix=fileKey):
logging.debug(
"[%s] Moving object %s from %s to %s object: ", workID,
obj.key, source, destination)
copy_source = {'Bucket': bucket.name, 'Key': obj.key}
response = s3.meta.client.copy_object(
CopySource=copy_source,
Bucket=bucket.name,
Key=obj.key,
StorageClass=s3StorageClass)
logging.debug("[%s] Object moved: ", workID, response)
OUTPUT = {
'result': 'success',
}
# If we need to move to or restore from archive, we need to run the whole gamut
elif 'archive' in [source, destination]: #Glacier
# Create Glacier object
# Create directory in working storage
subDir = parseHelper.createDir(workingStorage, fileKey)
# Pull down from glacier
if source == 'archive':
logging.info("[%s] Moving asset from Glacier", workID)
else:
logging.info("[%s] Begin moving objects to Glacier",
workID)
logging.info("[%s] Begin object download", workID)
# Download object to the working storage subdirectory
# Upload files back up to the same fileKey (this takes Accounts into consideration as well)
for obj in bucket.objects.filter(Prefix=fileKey):
logging.info(
"[%s] Downloading %s to temporary storage", workID,
obj.key)
fileName = os.path.join(workingStorage, obj.key)
if not os.path.exists(os.path.dirname(fileName)):
try:
os.makedirs(os.path.dirname(fileName))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
s3.Object(bucket.name, obj.key).download_file(
fileName) # Create directories as needed here
logging.info("[%s] Begin object upload to glacier", workID)
# Output needs the temporary storage location to clean up
# cleanUpLandingPads expects an ASSET (e.g., /Assets/working/file.ext), and not just a path. We will provide a dummy asset
OUTPUT = {
'result': 'success',
'asset': '%sdummy.file' % (subDir)
}
AUDIT = {}
AUDIT['User'] = '******'
AUDIT['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%S+0000",
time.gmtime())
AUDIT['Action'] = 'Asset moved from %s from %s' % (source,
destination)
AUDIT['Notes'] = workID
# Add the Audit Dictionary to a list so that we can append it
aLIST = []
aLIST.append(AUDIT)
updateExpression = 'set File_Location = :d, Audit = list_append(Audit, :a)'
expressionValues = {':d': destination, ':a': aLIST}
# Call the update function
logging.debug("[%s] Updating the asset location and history: %s",
workID, destination)
response = databaseHelper.updateEntry(dbPrimaryKey,
updateExpression,
expressionValues)
OUTPUT.update(INPUT)
swf.respond_activity_task_completed(taskToken=taskToken,
result=json.dumps(OUTPUT))
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
ARN = "arn:aws:states:us-east-1:497940546915:activity:createThumbnailFromImage"
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SFN boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
sfn = boto3.client('stepfunctions', config=botoConfig)
logging.debug("Created SFN boto client: %s", sfn)
while True:
task = sfn.get_activity_task(activityArn=ARN,
workerName='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['ResponseMetadata']['RequestId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
# Take the thumbnail 25% through the video
scale = "640x360"
newDir = "thumbnails"
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
outfile = '%s_thumbnail.jpg' % (fileName)
cmd = [
'convert', asset, '-format', 'jpg', '-thumbnail', scale,
'-auto-orient',
'%s/%s' % (subDir, outfile)
]
logging.debug("[%s] Execute image thumbnail creation: %s", workID,
cmd)
try:
output = subprocess.check_output(cmd)
# Start setting the parameters needed to update the thumbnail
key = dbPrimaryKey
updateExpression = 'set thumbnail = :t'
expressionValues = {':t': '/%s/%s' % (newDir, outfile)}
logging.debug("[%s] Update thumbnail value", workID)
# Call the update function
response = databaseHelper.updateEntry(key, updateExpression,
expressionValues)
OUTPUT = {
'tool': output,
'dbPrimaryKey': dbPrimaryKey,
'assetClass': INPUT['assetClass'],
'asset': asset,
}
sfn.send_task_success(taskToken=taskToken,
output=json.dumps(OUTPUT))
# We should catch other errors here
except subprocess.CalledProcessError as err:
result = {
'reason': 'THB-0001_Error in image thumbnail creation',
'detail': str(err)
}
logging.error("%s", result)
sfn.send_task_failure(taskToken=taskToken,
error=json.dumps(result['reason']),
cause=json.dumps(result['detail']))
logging.info("[%s] %s Complete", workID, taskName)