def main(args): ARN = "arn:aws:states:us-east-1:497940546915:activity:identifyAssetClass" taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SFN boto client") botoConfig = Config( connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect sfn = boto3.client('stepfunctions', config=botoConfig) logging.debug("Created SFN boto client: %s", sfn) # Should this be moved to the loop? Dyanmically change the item? EXT = loadExts() logging.info("Extension listing: %s", EXT) while True: task = sfn.get_activity_task(activityArn=ARN, workerName='%s-01' % (taskName)) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: logging.info(task) taskToken = task['taskToken'] workID = task['ResponseMetadata']['RequestId'] logging.info("[%s] New request for %s", workID, taskName) logging.debug("[%s] Input: %s", workID, task['input']) parameters = json.loads(task['input']) asset = parameters['asset'] # get the extensions filePath, fileName, fileExt = parseHelper.splitFilename(asset) # get the AssetClass # Extension will return with the period. We must nix this from the front try: assetClass = EXT[fileExt[1:].lower()] except KeyError: assetClass = "Other" logging.debug("[%s] File extension NOT found in list: %s", taskToken, fileExt) # metadata and asset are passthrough OUTPUT = { 'assetClass': assetClass, 'metadata': parameters['metadata'], 'asset': asset, } sfn.send_task_success(taskToken=taskToken, output=json.dumps(OUTPUT)) logging.info("[%s] %s Complete", workID, taskName)
def main(args): DOMAIN = 'ITD' TASKLIST = 'default' VERSION = '1' taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SWF boto client") botoConfig = Config(connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect swf = boto3.client('swf', config=botoConfig) logging.debug("Created SWF boto client: %s", swf) while True: task = swf.poll_for_activity_task( domain=DOMAIN, taskList={'name': taskName}, identity='%s-01' %(taskName) ) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: taskToken = task['taskToken'] workID = task['workflowExecution']['workflowId'] logging.info("[%s] New request for %s", workID, taskName) INPUT = json.loads(task['input']) asset = INPUT['asset'] filePath, fileName, fileExt = parseHelper.splitFilename(asset) logging.debug("[%s] Begin file tree deletion: %s", workID, filePath) shutil.rmtree(filePath) logging.debug("[%s] Completed file tree deletion: %s", workID, filePath) OUTPUT = { 'result' : 'success', } # As this activitiy is used by multiple workflows, we want to pass the INPUT parameters back OUTPUT.update(INPUT) swf.respond_activity_task_completed( taskToken = taskToken, result = json.dumps(OUTPUT) ) logging.info("[%s] %s Complete", workID, taskName)
def processFile(asset, METADATA, assetClass): # For each asset we need # Account # User # DateTime Added # Sha1 # Filename # General Metadata # Audio Track # Video Track sha1 = parseHelper.computeChecksum(asset) (filePath, fileName, fileExt) = parseHelper.splitFilename(asset) # -j is used for JSON # -d formats the dates into standard UTC # -c formats the GPS to decimal cmd = [ 'exiftool', '-j', '-d', '"%Y-%m-%dT%H:%M:%S+0000"', '-c', '"%+.8f"', asset ] output = subprocess.check_output(cmd) G, I = parseMediaToDict(output) # Start Constructing Document DOC = {} DOC['Filename'] = fileName DOC['Extension'] = fileExt DOC['Asset_Class'] = assetClass DOC['Checksum'] = sha1 DOC['Imported_Time'] = time.strftime("%Y-%m-%dT%H:%M:%S+0000", time.gmtime()) DOC['General'] = G DOC['Image'] = I DOC['File_Location'] = 'working' # Run a small post processing for GPS. We need to turn lat/lon into a "location point" for elastic search indexing try: LOC = { 'lat': DOC['General']['Latitude'], 'lon': DOC['General']['Longitude'] } DOC['General']['deviceLocation'] = LOC except KeyError: pass for m in METADATA: DOC[m] = METADATA[m] return DOC
def main(args): ARN = "arn:aws:states:us-east-1:497940546915:activity:cleanUpLandingPad" taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SFN boto client") botoConfig = Config( connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect sfn = boto3.client('stepfunctions', config=botoConfig) logging.debug("Created SFN boto client: %s", sfn) while True: task = sfn.get_activity_task(activityArn=ARN, workerName='%s-01' % (taskName)) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: taskToken = task['taskToken'] workID = task['ResponseMetadata']['RequestId'] logging.info("[%s] New request for %s", workID, taskName) INPUT = json.loads(task['input']) asset = INPUT['asset'] filePath, fileName, fileExt = parseHelper.splitFilename(asset) logging.debug("[%s] Begin file tree deletion: %s", workID, filePath) shutil.rmtree(filePath) logging.debug("[%s] Completed file tree deletion: %s", workID, filePath) OUTPUT = { 'result': 'success', } # As this activitiy is used by multiple workflows, we want to pass the INPUT parameters back OUTPUT.update(INPUT) sfn.send_task_success(taskToken=taskToken, output=json.dumps(OUTPUT)) logging.info("[%s] %s Complete", workID, taskName)
def main(args): ARN = "arn:aws:states:us-east-1:497940546915:activity:transcodeVideo" taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SFN boto client") botoConfig = Config( connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect sfn = boto3.client('stepfunctions', config=botoConfig) logging.debug("Created SFN boto client: %s", sfn) while True: task = sfn.get_activity_task(activityArn=ARN, workerName='%s-01' % (taskName)) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: taskToken = task['taskToken'] workID = task['ResponseMetadata']['RequestId'] logging.info("[%s] New request for %s", workID, taskName) INPUT = json.loads(task['input']) asset = INPUT['asset'] dbPrimaryKey = INPUT['dbPrimaryKey'] # Take the thumbnail 25% through the video newDir = "converted" (filePath, fileName, fileExt) = parseHelper.splitFilename(asset) subDir = parseHelper.createDir(filePath, newDir) # We require the %d to keep the file names incremented # Note that we need to escape the percentage sign by using another %, hence the double % outfile = '%s_PDL.mp4' % (fileName) cmd = [ 'ffmpeg', '-y', '-i', asset, '-c:v', 'libx264', '-preset', 'fast', '-crf', '22', '-c:a', 'aac', '-loglevel', 'fatal', '%s/%s' % (subDir, outfile) ] logging.debug("[%s] Execute video transcoding PDL creation: %s", workID, cmd) try: output = subprocess.check_output(cmd) OUTPUT = { 'tool': output, 'dbPrimaryKey': dbPrimaryKey, 'assetClass': INPUT['assetClass'], 'asset': asset, } logging.debug("[%s] Update PDL value", workID) updateExpression = 'set PDL = :t' expressionValues = { ':t': '/%s/%s' % (newDir, outfile), } response = databaseHelper.updateEntry(dbPrimaryKey, updateExpression, expressionValues) sfn.send_task_success(taskToken=taskToken, output=json.dumps(OUTPUT)) # We should catch other errors here except subprocess.CalledProcessError as err: logger.log('E', err.returncode) result = { 'reason': 'TRC-0001_Error in MP4 conversation', 'detail': str(err) } logging.error("%s", result) sfn.send_task_failure(taskToken=taskToken, error=json.dumps(result['reason']), cause=json.dumps(result['detail'])) logging.info("[%s] %s Complete", workID, taskName)
def main(args): DOMAIN = 'ITD' TASKLIST = 'default' VERSION = '1' taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SWF boto client") botoConfig = Config( connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect swf = boto3.client('swf', config=botoConfig) logging.debug("Created SWF boto client: %s", swf) while True: task = swf.poll_for_activity_task(domain=DOMAIN, taskList={'name': taskName}, identity='%s-01' % (taskName)) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: taskToken = task['taskToken'] workID = task['workflowExecution']['workflowId'] logging.info("[%s] New request for %s", workID, taskName) INPUT = json.loads(task['input']) asset = INPUT['asset'] dbPrimaryKey = INPUT['dbPrimaryKey'] # Take the thumbnail 25% through the video scale = "640x360" newDir = "thumbnails" (filePath, fileName, fileExt) = parseHelper.splitFilename(asset) subDir = parseHelper.createDir(filePath, newDir) outfile = '%s_thumbnail.jpg' % (fileName) cmd = [ 'convert', asset, '-format', 'jpg', '-thumbnail', scale, '-auto-orient', '%s/%s' % (subDir, outfile) ] logging.debug("[%s] Execute image thumbnail creation: %s", workID, cmd) try: output = subprocess.check_output(cmd) # Start setting the parameters needed to update the thumbnail key = dbPrimaryKey updateExpression = 'set thumbnail = :t' expressionValues = {':t': '/%s/%s' % (newDir, outfile)} logging.debug("[%s] Update thumbnail value", workID) # Call the update function response = databaseHelper.updateEntry(key, updateExpression, expressionValues) OUTPUT = { 'tool': output, 'dbPrimaryKey': dbPrimaryKey, 'assetClass': INPUT['assetClass'], 'asset': asset, } swf.respond_activity_task_completed(taskToken=taskToken, result=json.dumps(OUTPUT)) # We should catch other errors here except subprocess.CalledProcessError as err: result = { 'reason': 'THB-0001_Error in image thumbnail creation', 'detail': str(err) } logging.error("%s", result) swf.respond_activity_task_failed( taskToken=taskToken, reason=json.dumps(result['reason']), details=json.dumps(result['detail'])) logging.info("[%s] %s Complete", workID, taskName)
def main(args): DOMAIN = 'ITD' TASKLIST = 'default' VERSION = '1' taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SWF boto client") botoConfig = Config( connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect swf = boto3.client('swf', config=botoConfig) logging.debug("Created SWF boto client: %s", swf) while True: task = swf.poll_for_activity_task(domain=DOMAIN, taskList={'name': taskName}, identity='%s-01' % (taskName)) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: taskToken = task['taskToken'] workID = task['workflowExecution']['workflowId'] logging.info("[%s] New request for %s", workID, taskName) INPUT = json.loads(task['input']) asset = INPUT['asset'] dbPrimaryKey = INPUT['dbPrimaryKey'] # Take the thumbnail 25% through the video #scale = "640x360" # Use the multipliers so that we don't distort vertical videos. This makes it generic. scale = "iw/3:ih/3" # 1/3 gives 1920 (HD) down to 640 fps = 1 # Set the number of frames to be once per second newDir = "thumbnails" (filePath, fileName, fileExt) = parseHelper.splitFilename(asset) subDir = parseHelper.createDir(filePath, newDir) # We require the %d to keep the file names incremented # Note that we need to escape the percentage sign by using another %, hence the double % outfile = '%s_thumbnail_%%d.jpg' % (fileName) vtt = '%s.vtt' % (fileName) # Parameters are # -y for # -i for Input # -vf, fps=1,scale= for the video filter stating we want to take every one second cmd = [ 'ffmpeg', '-y', '-i', asset, '-vf', 'fps=%s,scale=%s' % (fps, scale), '-loglevel', 'fatal', '%s/%s' % (subDir, outfile) ] logging.debug("[%s] Execute video thumbnail creation: %s", workID, cmd) try: output = subprocess.check_output(cmd) # Start setting the parameters needed to update the thumbnail # Comment block is staying for reference sake '''# Call the update function # The "thumbnails" map will need to be created if it doesn't exist (Note: It shouldn't at this point) # A validation exception will be thrown, and when this is thrown, we will create an empty map and try it again try: response = databaseHelper.updateEntry(key, updateExpression, expressionValues) except botocore.exceptions.ClientError as err: if err.response['Error']['Code'] == 'ValidationException': response = databaseHelper.updateEntry(key, 'set thumbnails = :t', {':t' : {}}) response = databaseHelper.updateEntry(key, updateExpression, expressionValues) ''' # After the thumbnails are created, we need to do two things: # OLD # 1. Create the storyboard object which is [http://docs.brightcove.com/en/perform/brightcove-player/guides/thumbnails-plugin.html#collectimages] # 1. Create the storyboard VTT file (https://support.jwplayer.com/customer/portal/articles/1407439-adding-preview-thumbnails) # 2. We also need to identify the thumbnail for the video which we will take a percentage of the way through the video #STORYBOARD = {} thumbnailTime = .25 # Pick the thumbnail that's 25% of the way through the video counter = 0 for thumb in os.listdir(subDir): if fnmatch.fnmatch( thumb, '*_thumbnail_*.jpg' ): # Match files in the directory that are the thumbnails #sequenceNum = thumb[thumb.rfind('_')+1:-4] # filename_thumbnail_$frame.jpg #STORYBOARD[sequenceNum] = {'src' : '/%s/%s' %(newDir, thumb) } counter = counter + 1 # Open the VTT file and write logging.debug("[%s] Writing VTT file: %s", workID, vtt) vttFile = open('%s/%s' % (subDir, vtt), 'w') vttFile.write("WEBVTT") # The counter represents how many files of FPS we have -- range is COUNTER*FPS --> (COUNTER+1)* fps # FPS references the frames per second so if we put (1/60), that means a frame EVERY MINUTE # Therefore, we need to invest the FPS # Use %02d to PAD the numbers baseURL = "https://dnt4vq51jg2tj.cloudfront.net" # There needs to be a better way then the full URL for i in range(0, counter): startSecond = i * (1 / fps) endSecond = (i + 1) * (1 / fps) startSpan = '%02d:%02d:%02d.000' % (startSecond / 3600, startSecond / 60 % 60, startSecond % 60) endSpan = '%02d:%02d:%02d.000' % ( endSecond / 3600, endSecond / 60 % 60, endSecond % 60) thumbSpan = '%s/%s/%s/%s_thumbnail_%d.jpg' % ( baseURL, fileName, newDir, fileName, i + 1) vttFile.write("\n\n%s --> %s\n%s" % (startSpan, endSpan, thumbSpan)) vttFile.close() logging.debug("[%s] Wrote VTT file: %s", workID, vtt) index = str(math.trunc(counter * thumbnailTime)) logging.debug("[%s] Key frame identified in index: %s", workID, index) updateExpression = 'set thumbnail = :t, storyboard = :s' thumbnail = '/%s/%s_thumbnail_%s.jpg' % (newDir, fileName, index) # THERE MUST BE A DYNAMIC WAY TO DO THIS BUT I DONT KNOW YET storyboard = '/%s/%s' % (newDir, vtt) '''expressionValues = { ':t' : STORYBOARD[index]['src'], ':s' : STORYBOARD }''' expressionValues = { ':t': thumbnail, ':s': storyboard, } logging.debug("[%s] Update thumbnail value", workID) response = databaseHelper.updateEntry(dbPrimaryKey, updateExpression, expressionValues) OUTPUT = { 'tool': output, 'dbPrimaryKey': dbPrimaryKey, 'assetClass': INPUT['assetClass'], 'asset': asset, } swf.respond_activity_task_completed(taskToken=taskToken, result=json.dumps(OUTPUT)) # We should catch other errors here except subprocess.CalledProcessError as err: result = { 'reason': 'THB-0002_Error in video thumbnail creation', 'detail': str(err) } logging.error("%s", result) swf.respond_activity_task_failed( taskToken=taskToken, reason=json.dumps(result['reason']), details=json.dumps(result['detail'])) logging.info("[%s] %s Complete", workID, taskName)
def transcodeTest(): asset = '/Assets/test/trim.6A7FE5FA-5321-4BF1-87D7-D26937142263_76ed9582-7da6-44ef-a0af-e22596297573/trim.6A7FE5FA-5321-4BF1-87D7-D26937142263_76ed9582-7da6-44ef-a0af-e22596297573.MOV' scale = "640x360" newDir = "thumbnails" thumbnailTime = .25 # Create thumbnail 25% of the way through the video fps = 1 (filePath, fileName, fileExt) = parseHelper.splitFilename(asset) subDir = parseHelper.createDir(filePath, newDir) # We require the %d to keep the file names incremented # Note that we need to escape the percentage sign by using another %, hence the double % outfile = '%s_thumbnail_%%d.jpg' % (fileName) vtt = '%s.vtt' % (fileName) cmd = ['ffmpeg' ,'-y' ,'-i', asset ,'-s', scale ,'-vf', 'fps=%s' % (fps) ,'%s/%s' %(subDir, outfile) ] output = subprocess.check_output(cmd) print output # After the thumbnails are created, we need to do two things: # 1. Find the thumbnail we want to display, and create the object # 2. Create the storyboard object which is [http://docs.brightcove.com/en/perform/brightcove-player/guides/thumbnails-plugin.html#collectimages] # "second" : { "src" : "thumbnail"} STORYBOARD = {} counter = 0 for thumb in os.listdir(subDir): if fnmatch.fnmatch(thumb, '*_thumbnail_*.jpg'): counter = counter + 1 # Open the VTT file and write vttFile = open('%s/%s' %(subDir, vtt), 'w') vttFile.write("WEBVTT") # The counter represents how many files of FPS we have -- range is COUNTER*FPS --> (COUNTER+1)* fps # FPS references the frames per second so if we put (1/60), that means a frame EVERY MINUTE # Therefore, we need to invest the FPS # Use %02d to PAD the numbers for i in range(0,counter): startSecond = i * (1/fps) endSecond = (i + 1) * (1/fps) startSpan = '%02d:%02d:%02d.000' % ( startSecond / 3600, startSecond / 60 % 60, startSecond % 60) endSpan = '%02d:%02d:%02d.000' % ( endSecond / 3600, endSecond / 60 % 60, endSecond % 60) thumbSpan = '%s/%s_thumbnail_%d.jpg' % (newDir, fileName,i + 1) vttFile.write("\n\n%s --> %s\n%s" % (startSpan, endSpan, thumbSpan)) vttFile.close() index = str(math.trunc(counter * thumbnailTime)) thumbnail = '%s/%s_thumbnail_%s.jpg' % (newDir, fileName,index) sha1 = '6e508f0ed03da2b98ceb091827b569877037d1ef' # Start setting the parameters needed to update the thumbnail key = { 'Checksum' : sha1, } updateExpression = 'set thumbnail = :t, storyboard = :s' expressionValues = { ':t' : thumbnail, ':s' : '%s/%s' %(newDir, vtt) }
def main(args): DOMAIN = 'ITD' TASKLIST = 'default' VERSION = '1' taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SWF boto client") botoConfig = Config(connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect swf = boto3.client('swf', config=botoConfig) logging.debug("Created SWF boto client: %s", swf) while True: task = swf.poll_for_activity_task( domain=DOMAIN, taskList={'name': taskName}, identity='%s-01' %(taskName) ) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: taskToken = task['taskToken'] workID = task['workflowExecution']['workflowId'] logging.info("[%s] New request for %s", workID, taskName) INPUT = json.loads(task['input']) asset = INPUT['asset'] dbPrimaryKey = INPUT['dbPrimaryKey'] BUCKETNAME = "schulerfiles" logging.debug("[%s] Creating S3 bucket boto client", workID) s3 = boto3.resource('s3') bucket = s3.Bucket(BUCKETNAME) logging.debug("[%s] Creating S3 bucket boto client", workID) filePath, fileName, fileExt = parseHelper.splitFilename(asset) # fileName will serve as the FOLDERNAME # ext isn't not needed # filePath needs to be broken up if we want to organize things by account #b = bucket.new_key(folder) for dirName, dirList, fileList in os.walk(filePath): logging.info("[%s] Distributing file: %s", workID, asset) key = os.path.split(dirName)[1] for f in fileList: upfile = os.path.join(dirName,f) s3key = os.path.join(fileName, key, f) logging.debug("[%s] Started uploading %s with key: %s", workID, upfile, s3key) bucket.upload_file(upfile, s3key, {'ServerSideEncryption' :'AES256' }) logging.debug("[%s] Completed uploading %s with key: %s", workID, upfile, s3key) # Start setting the parameters needed to update the thumbnail updateExpression = 'set File_Location = :d' expressionValues = { ':d' : 'CDN' } # Call the update function logging.debug("[%s] Setting location to CDN for checksum: %s", workID, dbPrimaryKey) response = databaseHelper.updateEntry(dbPrimaryKey, updateExpression, expressionValues) OUTPUT = { 'result' : 'success', 'dbPrimaryKey' : dbPrimaryKey, 'assetClass' : INPUT['assetClass'], 'asset' : asset, } swf.respond_activity_task_completed( taskToken = taskToken, result = json.dumps(OUTPUT) ) logging.info("[%s] %s Complete", workID, taskName)
def main(args): ARN = "arn:aws:states:us-east-1:497940546915:activity:distributeToS3" taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SFN boto client") botoConfig = Config(connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect sfn = boto3.client('stepfunctions', config=botoConfig) logging.debug("Created SFN boto client: %s", sfn) while True: task = sfn.get_activity_task( activityArn=ARN, workerName='%s-01' %(taskName) ) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: taskToken = task['taskToken'] workID = task['ResponseMetadata']['RequestId'] logging.info("[%s] New request for %s", workID, taskName) # When two parallel tasks complete (like in the video section), we'll have two inputs # The try / except will handle this by assinging the input array the first of the series INPUT = json.loads(task['input']) try: asset = INPUT['asset'] except TypeError: INPUT = INPUT[0] asset = INPUT['asset'] dbPrimaryKey = INPUT['dbPrimaryKey'] BUCKETNAME = "schulerfiles" logging.debug("[%s] Creating S3 bucket boto client", workID) s3 = boto3.resource('s3') bucket = s3.Bucket(BUCKETNAME) logging.debug("[%s] Creating S3 bucket boto client", workID) filePath, fileName, fileExt = parseHelper.splitFilename(asset) # fileName will serve as the FOLDERNAME # ext isn't not needed # filePath needs to be broken up if we want to organize things by account #b = bucket.new_key(folder) for dirName, dirList, fileList in os.walk(filePath): logging.info("[%s] Distributing file: %s", workID, asset) key = os.path.split(dirName)[1] for f in fileList: upfile = os.path.join(dirName,f) s3key = os.path.join(fileName, key, f) logging.debug("[%s] Started uploading %s with key: %s", workID, upfile, s3key) bucket.upload_file(upfile, s3key, {'ServerSideEncryption' :'AES256' }) logging.debug("[%s] Completed uploading %s with key: %s", workID, upfile, s3key) # Start setting the parameters needed to update the thumbnail updateExpression = 'set File_Location = :d' expressionValues = { ':d' : 'CDN' } # Call the update function logging.debug("[%s] Setting location to CDN for checksum: %s", workID, dbPrimaryKey) response = databaseHelper.updateEntry(dbPrimaryKey, updateExpression, expressionValues) OUTPUT = { 'result' : 'success', 'dbPrimaryKey' : dbPrimaryKey, 'assetClass' : INPUT['assetClass'], 'asset' : asset, } sfn.send_task_success( taskToken=taskToken, output=json.dumps(OUTPUT) ) logging.info("[%s] %s Complete", workID, taskName)
def main(args): DOMAIN = 'ITD' TASKLIST = 'default' VERSION = '1' taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SWF boto client") botoConfig = Config(connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect swf = boto3.client('swf', config=botoConfig) logging.debug("Created SWF boto client: %s", swf) # Should this be moved to the loop? Dyanmically change the item? EXT = loadExts() logging.info("Extension listing: %s", EXT) while True: task = swf.poll_for_activity_task( domain=DOMAIN, taskList={'name': taskName}, identity='%s-01' %(taskName) ) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: logging.info(task) taskToken = task['taskToken'] workID = task['workflowExecution']['workflowId'] logging.info("[%s] New request for %s", workID, taskName) logging.debug("[%s] Input: %s", workID, task['input']) parameters = json.loads(task['input']) asset = parameters['asset'] # get the extensions filePath, fileName, fileExt = parseHelper.splitFilename(asset) # get the AssetClass # Extension will return with the period. We must nix this from the front try: assetClass = EXT[fileExt[1:].lower()] except KeyError: assetClass = "Other" logging.debug("[%s] File extension NOT found in list: %s", taskToken, fileExt) # metadata and asset are passthrough result = { 'assetClass' : assetClass, 'metadata' : parameters['metadata'], 'asset' : asset, } swf.respond_activity_task_completed( taskToken=task['taskToken'], result=json.dumps(result) ) logging.info("[%s] %s Complete", workID, taskName)
def main(args): ARN = "arn:aws:states:us-east-1:497940546915:activity:createThumbnailFromImage" taskName = os.path.basename(__file__)[:-3] logging.config.fileConfig('/Assets/sharedLibraries/logging_config.ini') logging.debug("Creating SFN boto client") botoConfig = Config( connect_timeout=50, read_timeout=70) # suggestion is the read is higher than connect sfn = boto3.client('stepfunctions', config=botoConfig) logging.debug("Created SFN boto client: %s", sfn) while True: task = sfn.get_activity_task(activityArn=ARN, workerName='%s-01' % (taskName)) if 'taskToken' not in task: logging.info("%s - Poll timed out, no new task. Repoll", taskName) # Run the operation else: taskToken = task['taskToken'] workID = task['ResponseMetadata']['RequestId'] logging.info("[%s] New request for %s", workID, taskName) INPUT = json.loads(task['input']) asset = INPUT['asset'] dbPrimaryKey = INPUT['dbPrimaryKey'] # Take the thumbnail 25% through the video scale = "640x360" newDir = "thumbnails" (filePath, fileName, fileExt) = parseHelper.splitFilename(asset) subDir = parseHelper.createDir(filePath, newDir) outfile = '%s_thumbnail.jpg' % (fileName) cmd = [ 'convert', asset, '-format', 'jpg', '-thumbnail', scale, '-auto-orient', '%s/%s' % (subDir, outfile) ] logging.debug("[%s] Execute image thumbnail creation: %s", workID, cmd) try: output = subprocess.check_output(cmd) # Start setting the parameters needed to update the thumbnail key = dbPrimaryKey updateExpression = 'set thumbnail = :t' expressionValues = {':t': '/%s/%s' % (newDir, outfile)} logging.debug("[%s] Update thumbnail value", workID) # Call the update function response = databaseHelper.updateEntry(key, updateExpression, expressionValues) OUTPUT = { 'tool': output, 'dbPrimaryKey': dbPrimaryKey, 'assetClass': INPUT['assetClass'], 'asset': asset, } sfn.send_task_success(taskToken=taskToken, output=json.dumps(OUTPUT)) # We should catch other errors here except subprocess.CalledProcessError as err: result = { 'reason': 'THB-0001_Error in image thumbnail creation', 'detail': str(err) } logging.error("%s", result) sfn.send_task_failure(taskToken=taskToken, error=json.dumps(result['reason']), cause=json.dumps(result['detail'])) logging.info("[%s] %s Complete", workID, taskName)