def _send_bucket_delete_list(objectList): print 'Attempting to delete list' content_bucket = _get_content_bucket() try: res = content_bucket.delete_objects(Delete={'Objects': objectList}) except ClientError as e: raise errors.PortalRequestError( 'Failed to delete list from bucket: {}'.format( e.response['Error']['Message']))
def _remove_entry(key_path): print 'Removing entry {}'.format(key_path) staging_table = _get_staging_table() try: response = staging_table.delete_item(Key={'FileName': key_path}) return True except ClientError as e: raise errors.PortalRequestError( 'Failed to remove entry from table: {}'.format( e.response['Error']['Message']))
def _get_channel_table(): if not hasattr(_get_channel_table, 'channel_table'): channel_table_name = CloudCanvas.get_setting('ChannelDataTable') dynamoresource = boto3.resource('dynamodb') _get_channel_table.channel_table = dynamoresource.Table( channel_table_name) if _get_channel_table.channel_table is None: raise errors.PortalRequestError('No Channel Table') return _get_channel_table.channel_table
def _remove_bucket_item(key_path): content_bucket = _get_content_bucket() print 'Attempting to delete object {}'.format(key_path) try: res = content_bucket.delete_objects( Delete={'Objects': [{ 'Key': key_path }]}) print 'Success: {}'.format(res) except ClientError as e: raise errors.PortalRequestError( 'Failed to remove item from bucket: {}'.format( e.response['Error']['Message']))
def _get_staging_table(): if not hasattr(_get_staging_table,'staging_table'): staging_table_name = CloudCanvas.get_setting('StagingTable') print 'Table name is {}'.format(staging_table_name) print 'attempting to get dynamodb resource from local region' dynamoresource = boto3.resource('dynamodb') _get_staging_table.staging_table = dynamoresource.Table(staging_table_name) if _get_staging_table.staging_table is None: raise errors.PortalRequestError('No Staging Table') return _get_staging_table.staging_table
def _get_content_bucket(): if not hasattr(_get_content_bucket,'content_bucket'): content_bucket_name = CloudCanvas.get_setting('StagingBucket') print 'Bucket name is {}'.format(content_bucket_name) print 'attempting to get S3 resource from local region' bucketResource = boto3.resource('s3', config=Config(signature_version='s3v4')) _get_content_bucket.content_bucket = bucketResource.Bucket(content_bucket_name) if _get_content_bucket.content_bucket is None: raise errors.PortalRequestError('No Content Bucket') return _get_content_bucket.content_bucket
def _get_bucket_content_list(): print 'Getting bucket content list' s3 = boto3.client('s3', config=Config(signature_version='s3v4')) content_bucket_name = CloudCanvas.get_setting('StagingBucket') nextMarker = 0 contentsList = [] while True: try: print 'Listing objects in {}'.format(content_bucket_name) res = s3.list_objects(Bucket=content_bucket_name, Marker=str(nextMarker)) thisList = res.get('Contents', []) contentsList += thisList print 'Appending items: {}'.format(thisList) if len(thisList) < get_list_objects_limit(): break nextMarker += get_list_objects_limit() except ClientError as e: raise errors.PortalRequestError( 'Failed to list bucket content: {}'.format( e.response['Error']['Message'])) return contentsList