def main(args):
ARN = "arn:aws:states:us-east-1:497940546915:activity:identifyAssetClass"
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SFN boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
sfn = boto3.client('stepfunctions', config=botoConfig)
logging.debug("Created SFN boto client: %s", sfn)
# Should this be moved to the loop? Dyanmically change the item?
EXT = loadExts()
logging.info("Extension listing: %s", EXT)
while True:
task = sfn.get_activity_task(activityArn=ARN,
workerName='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
logging.info(task)
taskToken = task['taskToken']
workID = task['ResponseMetadata']['RequestId']
logging.info("[%s] New request for %s", workID, taskName)
logging.debug("[%s] Input: %s", workID, task['input'])
parameters = json.loads(task['input'])
asset = parameters['asset']
# get the extensions
filePath, fileName, fileExt = parseHelper.splitFilename(asset)
# get the AssetClass
# Extension will return with the period. We must nix this from the front
try:
assetClass = EXT[fileExt[1:].lower()]
except KeyError:
assetClass = "Other"
logging.debug("[%s] File extension NOT found in list: %s",
taskToken, fileExt)
# metadata and asset are passthrough
OUTPUT = {
'assetClass': assetClass,
'metadata': parameters['metadata'],
'asset': asset,
}
sfn.send_task_success(taskToken=taskToken,
output=json.dumps(OUTPUT))
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
DOMAIN = 'ITD'
TASKLIST = 'default'
VERSION = '1'
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SWF boto client")
botoConfig = Config(connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect
swf = boto3.client('swf', config=botoConfig)
logging.debug("Created SWF boto client: %s", swf)
while True:
task = swf.poll_for_activity_task(
domain=DOMAIN,
taskList={'name': taskName},
identity='%s-01' %(taskName)
)
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['workflowExecution']['workflowId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
filePath, fileName, fileExt = parseHelper.splitFilename(asset)
logging.debug("[%s] Begin file tree deletion: %s", workID, filePath)
shutil.rmtree(filePath)
logging.debug("[%s] Completed file tree deletion: %s", workID, filePath)
OUTPUT = {
'result' : 'success',
}
# As this activitiy is used by multiple workflows, we want to pass the INPUT parameters back
OUTPUT.update(INPUT)
swf.respond_activity_task_completed(
taskToken = taskToken,
result = json.dumps(OUTPUT)
)
logging.info("[%s] %s Complete", workID, taskName)
def processFile(asset, METADATA, assetClass):
# For each asset we need
# Account
# User
# DateTime Added
# Sha1
# Filename
# General Metadata
# Audio Track
# Video Track
sha1 = parseHelper.computeChecksum(asset)
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
# -j is used for JSON
# -d formats the dates into standard UTC
# -c formats the GPS to decimal
cmd = [
'exiftool', '-j', '-d', '"%Y-%m-%dT%H:%M:%S+0000"', '-c', '"%+.8f"',
asset
]
output = subprocess.check_output(cmd)
G, I = parseMediaToDict(output)
# Start Constructing Document
DOC = {}
DOC['Filename'] = fileName
DOC['Extension'] = fileExt
DOC['Asset_Class'] = assetClass
DOC['Checksum'] = sha1
DOC['Imported_Time'] = time.strftime("%Y-%m-%dT%H:%M:%S+0000",
time.gmtime())
DOC['General'] = G
DOC['Image'] = I
DOC['File_Location'] = 'working'
# Run a small post processing for GPS. We need to turn lat/lon into a "location point" for elastic search indexing
try:
LOC = {
'lat': DOC['General']['Latitude'],
'lon': DOC['General']['Longitude']
}
DOC['General']['deviceLocation'] = LOC
except KeyError:
pass
for m in METADATA:
DOC[m] = METADATA[m]
return DOC
def main(args):
ARN = "arn:aws:states:us-east-1:497940546915:activity:cleanUpLandingPad"
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SFN boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
sfn = boto3.client('stepfunctions', config=botoConfig)
logging.debug("Created SFN boto client: %s", sfn)
while True:
task = sfn.get_activity_task(activityArn=ARN,
workerName='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['ResponseMetadata']['RequestId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
filePath, fileName, fileExt = parseHelper.splitFilename(asset)
logging.debug("[%s] Begin file tree deletion: %s", workID,
filePath)
shutil.rmtree(filePath)
logging.debug("[%s] Completed file tree deletion: %s", workID,
filePath)
OUTPUT = {
'result': 'success',
}
# As this activitiy is used by multiple workflows, we want to pass the INPUT parameters back
OUTPUT.update(INPUT)
sfn.send_task_success(taskToken=taskToken,
output=json.dumps(OUTPUT))
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
ARN = "arn:aws:states:us-east-1:497940546915:activity:transcodeVideo"
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SFN boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
sfn = boto3.client('stepfunctions', config=botoConfig)
logging.debug("Created SFN boto client: %s", sfn)
while True:
task = sfn.get_activity_task(activityArn=ARN,
workerName='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['ResponseMetadata']['RequestId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
# Take the thumbnail 25% through the video
newDir = "converted"
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
# We require the %d to keep the file names incremented
# Note that we need to escape the percentage sign by using another %, hence the double %
outfile = '%s_PDL.mp4' % (fileName)
cmd = [
'ffmpeg', '-y', '-i', asset, '-c:v', 'libx264', '-preset',
'fast', '-crf', '22', '-c:a', 'aac', '-loglevel', 'fatal',
'%s/%s' % (subDir, outfile)
]
logging.debug("[%s] Execute video transcoding PDL creation: %s",
workID, cmd)
try:
output = subprocess.check_output(cmd)
OUTPUT = {
'tool': output,
'dbPrimaryKey': dbPrimaryKey,
'assetClass': INPUT['assetClass'],
'asset': asset,
}
logging.debug("[%s] Update PDL value", workID)
updateExpression = 'set PDL = :t'
expressionValues = {
':t': '/%s/%s' % (newDir, outfile),
}
response = databaseHelper.updateEntry(dbPrimaryKey,
updateExpression,
expressionValues)
sfn.send_task_success(taskToken=taskToken,
output=json.dumps(OUTPUT))
# We should catch other errors here
except subprocess.CalledProcessError as err:
logger.log('E', err.returncode)
result = {
'reason': 'TRC-0001_Error in MP4 conversation',
'detail': str(err)
}
logging.error("%s", result)
sfn.send_task_failure(taskToken=taskToken,
error=json.dumps(result['reason']),
cause=json.dumps(result['detail']))
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
DOMAIN = 'ITD'
TASKLIST = 'default'
VERSION = '1'
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SWF boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
swf = boto3.client('swf', config=botoConfig)
logging.debug("Created SWF boto client: %s", swf)
while True:
task = swf.poll_for_activity_task(domain=DOMAIN,
taskList={'name': taskName},
identity='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['workflowExecution']['workflowId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
# Take the thumbnail 25% through the video
scale = "640x360"
newDir = "thumbnails"
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
outfile = '%s_thumbnail.jpg' % (fileName)
cmd = [
'convert', asset, '-format', 'jpg', '-thumbnail', scale,
'-auto-orient',
'%s/%s' % (subDir, outfile)
]
logging.debug("[%s] Execute image thumbnail creation: %s", workID,
cmd)
try:
output = subprocess.check_output(cmd)
# Start setting the parameters needed to update the thumbnail
key = dbPrimaryKey
updateExpression = 'set thumbnail = :t'
expressionValues = {':t': '/%s/%s' % (newDir, outfile)}
logging.debug("[%s] Update thumbnail value", workID)
# Call the update function
response = databaseHelper.updateEntry(key, updateExpression,
expressionValues)
OUTPUT = {
'tool': output,
'dbPrimaryKey': dbPrimaryKey,
'assetClass': INPUT['assetClass'],
'asset': asset,
}
swf.respond_activity_task_completed(taskToken=taskToken,
result=json.dumps(OUTPUT))
# We should catch other errors here
except subprocess.CalledProcessError as err:
result = {
'reason': 'THB-0001_Error in image thumbnail creation',
'detail': str(err)
}
logging.error("%s", result)
swf.respond_activity_task_failed(
taskToken=taskToken,
reason=json.dumps(result['reason']),
details=json.dumps(result['detail']))
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
DOMAIN = 'ITD'
TASKLIST = 'default'
VERSION = '1'
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SWF boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
swf = boto3.client('swf', config=botoConfig)
logging.debug("Created SWF boto client: %s", swf)
while True:
task = swf.poll_for_activity_task(domain=DOMAIN,
taskList={'name': taskName},
identity='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['workflowExecution']['workflowId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
# Take the thumbnail 25% through the video
#scale = "640x360"
# Use the multipliers so that we don't distort vertical videos. This makes it generic.
scale = "iw/3:ih/3" # 1/3 gives 1920 (HD) down to 640
fps = 1 # Set the number of frames to be once per second
newDir = "thumbnails"
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
# We require the %d to keep the file names incremented
# Note that we need to escape the percentage sign by using another %, hence the double %
outfile = '%s_thumbnail_%%d.jpg' % (fileName)
vtt = '%s.vtt' % (fileName)
# Parameters are
# -y for
# -i for Input
# -vf, fps=1,scale= for the video filter stating we want to take every one second
cmd = [
'ffmpeg', '-y', '-i', asset, '-vf',
'fps=%s,scale=%s' % (fps, scale), '-loglevel', 'fatal',
'%s/%s' % (subDir, outfile)
]
logging.debug("[%s] Execute video thumbnail creation: %s", workID,
cmd)
try:
output = subprocess.check_output(cmd)
# Start setting the parameters needed to update the thumbnail
# Comment block is staying for reference sake
'''# Call the update function
# The "thumbnails" map will need to be created if it doesn't exist (Note: It shouldn't at this point)
# A validation exception will be thrown, and when this is thrown, we will create an empty map and try it again
try:
response = databaseHelper.updateEntry(key, updateExpression, expressionValues)
except botocore.exceptions.ClientError as err:
if err.response['Error']['Code'] == 'ValidationException':
response = databaseHelper.updateEntry(key, 'set thumbnails = :t', {':t' : {}})
response = databaseHelper.updateEntry(key, updateExpression, expressionValues)
'''
# After the thumbnails are created, we need to do two things:
# OLD # 1. Create the storyboard object which is [http://docs.brightcove.com/en/perform/brightcove-player/guides/thumbnails-plugin.html#collectimages]
# 1. Create the storyboard VTT file (https://support.jwplayer.com/customer/portal/articles/1407439-adding-preview-thumbnails)
# 2. We also need to identify the thumbnail for the video which we will take a percentage of the way through the video
#STORYBOARD = {}
thumbnailTime = .25 # Pick the thumbnail that's 25% of the way through the video
counter = 0
for thumb in os.listdir(subDir):
if fnmatch.fnmatch(
thumb, '*_thumbnail_*.jpg'
): # Match files in the directory that are the thumbnails
#sequenceNum = thumb[thumb.rfind('_')+1:-4] # filename_thumbnail_$frame.jpg
#STORYBOARD[sequenceNum] = {'src' : '/%s/%s' %(newDir, thumb) }
counter = counter + 1
# Open the VTT file and write
logging.debug("[%s] Writing VTT file: %s", workID, vtt)
vttFile = open('%s/%s' % (subDir, vtt), 'w')
vttFile.write("WEBVTT")
# The counter represents how many files of FPS we have -- range is COUNTER*FPS --> (COUNTER+1)* fps
# FPS references the frames per second so if we put (1/60), that means a frame EVERY MINUTE
# Therefore, we need to invest the FPS
# Use %02d to PAD the numbers
baseURL = "https://dnt4vq51jg2tj.cloudfront.net" # There needs to be a better way then the full URL
for i in range(0, counter):
startSecond = i * (1 / fps)
endSecond = (i + 1) * (1 / fps)
startSpan = '%02d:%02d:%02d.000' % (startSecond / 3600,
startSecond / 60 % 60,
startSecond % 60)
endSpan = '%02d:%02d:%02d.000' % (
endSecond / 3600, endSecond / 60 % 60, endSecond % 60)
thumbSpan = '%s/%s/%s/%s_thumbnail_%d.jpg' % (
baseURL, fileName, newDir, fileName, i + 1)
vttFile.write("\n\n%s --> %s\n%s" %
(startSpan, endSpan, thumbSpan))
vttFile.close()
logging.debug("[%s] Wrote VTT file: %s", workID, vtt)
index = str(math.trunc(counter * thumbnailTime))
logging.debug("[%s] Key frame identified in index: %s", workID,
index)
updateExpression = 'set thumbnail = :t, storyboard = :s'
thumbnail = '/%s/%s_thumbnail_%s.jpg' % (newDir, fileName,
index)
# THERE MUST BE A DYNAMIC WAY TO DO THIS BUT I DONT KNOW YET
storyboard = '/%s/%s' % (newDir, vtt)
'''expressionValues = {
':t' : STORYBOARD[index]['src'],
':s' : STORYBOARD
}'''
expressionValues = {
':t': thumbnail,
':s': storyboard,
}
logging.debug("[%s] Update thumbnail value", workID)
response = databaseHelper.updateEntry(dbPrimaryKey,
updateExpression,
expressionValues)
OUTPUT = {
'tool': output,
'dbPrimaryKey': dbPrimaryKey,
'assetClass': INPUT['assetClass'],
'asset': asset,
}
swf.respond_activity_task_completed(taskToken=taskToken,
result=json.dumps(OUTPUT))
# We should catch other errors here
except subprocess.CalledProcessError as err:
result = {
'reason': 'THB-0002_Error in video thumbnail creation',
'detail': str(err)
}
logging.error("%s", result)
swf.respond_activity_task_failed(
taskToken=taskToken,
reason=json.dumps(result['reason']),
details=json.dumps(result['detail']))
logging.info("[%s] %s Complete", workID, taskName)
def transcodeTest():
asset = '/Assets/test/trim.6A7FE5FA-5321-4BF1-87D7-D26937142263_76ed9582-7da6-44ef-a0af-e22596297573/trim.6A7FE5FA-5321-4BF1-87D7-D26937142263_76ed9582-7da6-44ef-a0af-e22596297573.MOV'
scale = "640x360"
newDir = "thumbnails"
thumbnailTime = .25 # Create thumbnail 25% of the way through the video
fps = 1
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
# We require the %d to keep the file names incremented
# Note that we need to escape the percentage sign by using another %, hence the double %
outfile = '%s_thumbnail_%%d.jpg' % (fileName)
vtt = '%s.vtt' % (fileName)
cmd = ['ffmpeg'
,'-y'
,'-i', asset
,'-s', scale
,'-vf', 'fps=%s' % (fps)
,'%s/%s' %(subDir, outfile)
]
output = subprocess.check_output(cmd)
print output
# After the thumbnails are created, we need to do two things:
# 1. Find the thumbnail we want to display, and create the object
# 2. Create the storyboard object which is [http://docs.brightcove.com/en/perform/brightcove-player/guides/thumbnails-plugin.html#collectimages]
# "second" : { "src" : "thumbnail"}
STORYBOARD = {}
counter = 0
for thumb in os.listdir(subDir):
if fnmatch.fnmatch(thumb, '*_thumbnail_*.jpg'):
counter = counter + 1
# Open the VTT file and write
vttFile = open('%s/%s' %(subDir, vtt), 'w')
vttFile.write("WEBVTT")
# The counter represents how many files of FPS we have -- range is COUNTER*FPS --> (COUNTER+1)* fps
# FPS references the frames per second so if we put (1/60), that means a frame EVERY MINUTE
# Therefore, we need to invest the FPS
# Use %02d to PAD the numbers
for i in range(0,counter):
startSecond = i * (1/fps)
endSecond = (i + 1) * (1/fps)
startSpan = '%02d:%02d:%02d.000' % ( startSecond / 3600, startSecond / 60 % 60, startSecond % 60)
endSpan = '%02d:%02d:%02d.000' % ( endSecond / 3600, endSecond / 60 % 60, endSecond % 60)
thumbSpan = '%s/%s_thumbnail_%d.jpg' % (newDir, fileName,i + 1)
vttFile.write("\n\n%s --> %s\n%s" % (startSpan, endSpan, thumbSpan))
vttFile.close()
index = str(math.trunc(counter * thumbnailTime))
thumbnail = '%s/%s_thumbnail_%s.jpg' % (newDir, fileName,index)
sha1 = '6e508f0ed03da2b98ceb091827b569877037d1ef'
# Start setting the parameters needed to update the thumbnail
key = {
'Checksum' : sha1,
}
updateExpression = 'set thumbnail = :t, storyboard = :s'
expressionValues = {
':t' : thumbnail,
':s' : '%s/%s' %(newDir, vtt)
}
def main(args):
DOMAIN = 'ITD'
TASKLIST = 'default'
VERSION = '1'
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SWF boto client")
botoConfig = Config(connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect
swf = boto3.client('swf', config=botoConfig)
logging.debug("Created SWF boto client: %s", swf)
while True:
task = swf.poll_for_activity_task(
domain=DOMAIN,
taskList={'name': taskName},
identity='%s-01' %(taskName)
)
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['workflowExecution']['workflowId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
BUCKETNAME = "schulerfiles"
logging.debug("[%s] Creating S3 bucket boto client", workID)
s3 = boto3.resource('s3')
bucket = s3.Bucket(BUCKETNAME)
logging.debug("[%s] Creating S3 bucket boto client", workID)
filePath, fileName, fileExt = parseHelper.splitFilename(asset)
# fileName will serve as the FOLDERNAME
# ext isn't not needed
# filePath needs to be broken up if we want to organize things by account
#b = bucket.new_key(folder)
for dirName, dirList, fileList in os.walk(filePath):
logging.info("[%s] Distributing file: %s", workID, asset)
key = os.path.split(dirName)[1]
for f in fileList:
upfile = os.path.join(dirName,f)
s3key = os.path.join(fileName, key, f)
logging.debug("[%s] Started uploading %s with key: %s", workID, upfile, s3key)
bucket.upload_file(upfile, s3key, {'ServerSideEncryption' :'AES256' })
logging.debug("[%s] Completed uploading %s with key: %s", workID, upfile, s3key)
# Start setting the parameters needed to update the thumbnail
updateExpression = 'set File_Location = :d'
expressionValues = {
':d' : 'CDN'
}
# Call the update function
logging.debug("[%s] Setting location to CDN for checksum: %s", workID, dbPrimaryKey)
response = databaseHelper.updateEntry(dbPrimaryKey, updateExpression, expressionValues)
OUTPUT = {
'result' : 'success',
'dbPrimaryKey' : dbPrimaryKey,
'assetClass' : INPUT['assetClass'],
'asset' : asset,
}
swf.respond_activity_task_completed(
taskToken = taskToken,
result = json.dumps(OUTPUT)
)
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
ARN = "arn:aws:states:us-east-1:497940546915:activity:distributeToS3"
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SFN boto client")
botoConfig = Config(connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect
sfn = boto3.client('stepfunctions', config=botoConfig)
logging.debug("Created SFN boto client: %s", sfn)
while True:
task = sfn.get_activity_task(
activityArn=ARN,
workerName='%s-01' %(taskName)
)
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['ResponseMetadata']['RequestId']
logging.info("[%s] New request for %s", workID, taskName)
# When two parallel tasks complete (like in the video section), we'll have two inputs
# The try / except will handle this by assinging the input array the first of the series
INPUT = json.loads(task['input'])
try:
asset = INPUT['asset']
except TypeError:
INPUT = INPUT[0]
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
BUCKETNAME = "schulerfiles"
logging.debug("[%s] Creating S3 bucket boto client", workID)
s3 = boto3.resource('s3')
bucket = s3.Bucket(BUCKETNAME)
logging.debug("[%s] Creating S3 bucket boto client", workID)
filePath, fileName, fileExt = parseHelper.splitFilename(asset)
# fileName will serve as the FOLDERNAME
# ext isn't not needed
# filePath needs to be broken up if we want to organize things by account
#b = bucket.new_key(folder)
for dirName, dirList, fileList in os.walk(filePath):
logging.info("[%s] Distributing file: %s", workID, asset)
key = os.path.split(dirName)[1]
for f in fileList:
upfile = os.path.join(dirName,f)
s3key = os.path.join(fileName, key, f)
logging.debug("[%s] Started uploading %s with key: %s", workID, upfile, s3key)
bucket.upload_file(upfile, s3key, {'ServerSideEncryption' :'AES256' })
logging.debug("[%s] Completed uploading %s with key: %s", workID, upfile, s3key)
# Start setting the parameters needed to update the thumbnail
updateExpression = 'set File_Location = :d'
expressionValues = {
':d' : 'CDN'
}
# Call the update function
logging.debug("[%s] Setting location to CDN for checksum: %s", workID, dbPrimaryKey)
response = databaseHelper.updateEntry(dbPrimaryKey, updateExpression, expressionValues)
OUTPUT = {
'result' : 'success',
'dbPrimaryKey' : dbPrimaryKey,
'assetClass' : INPUT['assetClass'],
'asset' : asset,
}
sfn.send_task_success(
taskToken=taskToken,
output=json.dumps(OUTPUT)
)
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
DOMAIN = 'ITD'
TASKLIST = 'default'
VERSION = '1'
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SWF boto client")
botoConfig = Config(connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect
swf = boto3.client('swf', config=botoConfig)
logging.debug("Created SWF boto client: %s", swf)
# Should this be moved to the loop? Dyanmically change the item?
EXT = loadExts()
logging.info("Extension listing: %s", EXT)
while True:
task = swf.poll_for_activity_task(
domain=DOMAIN,
taskList={'name': taskName},
identity='%s-01' %(taskName)
)
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
logging.info(task)
taskToken = task['taskToken']
workID = task['workflowExecution']['workflowId']
logging.info("[%s] New request for %s", workID, taskName)
logging.debug("[%s] Input: %s", workID, task['input'])
parameters = json.loads(task['input'])
asset = parameters['asset']
# get the extensions
filePath, fileName, fileExt = parseHelper.splitFilename(asset)
# get the AssetClass
# Extension will return with the period. We must nix this from the front
try:
assetClass = EXT[fileExt[1:].lower()]
except KeyError:
assetClass = "Other"
logging.debug("[%s] File extension NOT found in list: %s", taskToken, fileExt)
# metadata and asset are passthrough
result = {
'assetClass' : assetClass,
'metadata' : parameters['metadata'],
'asset' : asset,
}
swf.respond_activity_task_completed(
taskToken=task['taskToken'],
result=json.dumps(result)
)
logging.info("[%s] %s Complete", workID, taskName)
def main(args):
ARN = "arn:aws:states:us-east-1:497940546915:activity:createThumbnailFromImage"
taskName = os.path.basename(__file__)[:-3]
logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini')
logging.debug("Creating SFN boto client")
botoConfig = Config(
connect_timeout=50,
read_timeout=70) # suggestion is the read is higher than connect
sfn = boto3.client('stepfunctions', config=botoConfig)
logging.debug("Created SFN boto client: %s", sfn)
while True:
task = sfn.get_activity_task(activityArn=ARN,
workerName='%s-01' % (taskName))
if 'taskToken' not in task:
logging.info("%s - Poll timed out, no new task. Repoll", taskName)
# Run the operation
else:
taskToken = task['taskToken']
workID = task['ResponseMetadata']['RequestId']
logging.info("[%s] New request for %s", workID, taskName)
INPUT = json.loads(task['input'])
asset = INPUT['asset']
dbPrimaryKey = INPUT['dbPrimaryKey']
# Take the thumbnail 25% through the video
scale = "640x360"
newDir = "thumbnails"
(filePath, fileName, fileExt) = parseHelper.splitFilename(asset)
subDir = parseHelper.createDir(filePath, newDir)
outfile = '%s_thumbnail.jpg' % (fileName)
cmd = [
'convert', asset, '-format', 'jpg', '-thumbnail', scale,
'-auto-orient',
'%s/%s' % (subDir, outfile)
]
logging.debug("[%s] Execute image thumbnail creation: %s", workID,
cmd)
try:
output = subprocess.check_output(cmd)
# Start setting the parameters needed to update the thumbnail
key = dbPrimaryKey
updateExpression = 'set thumbnail = :t'
expressionValues = {':t': '/%s/%s' % (newDir, outfile)}
logging.debug("[%s] Update thumbnail value", workID)
# Call the update function
response = databaseHelper.updateEntry(key, updateExpression,
expressionValues)
OUTPUT = {
'tool': output,
'dbPrimaryKey': dbPrimaryKey,
'assetClass': INPUT['assetClass'],
'asset': asset,
}
sfn.send_task_success(taskToken=taskToken,
output=json.dumps(OUTPUT))
# We should catch other errors here
except subprocess.CalledProcessError as err:
result = {
'reason': 'THB-0001_Error in image thumbnail creation',
'detail': str(err)
}
logging.error("%s", result)
sfn.send_task_failure(taskToken=taskToken,
error=json.dumps(result['reason']),
cause=json.dumps(result['detail']))
logging.info("[%s] %s Complete", workID, taskName)
