def get_bucket(bucket, aws_access_key_id, aws_secret_access_key, **kwargs):
import boto
bucket = get_bucket_name(bucket)
connection = get_boto_connection(aws_access_key_id, aws_secret_access_key,
bucketname=bucket)
try:
return connection.get_bucket(bucket)
except boto.exception.S3ResponseError:
return connection.get_bucket(bucket, validate=False)
def get_bucket(bucket, aws_access_key_id, aws_secret_access_key, **kwargs):
import boto
bucket = get_bucket_name(bucket)
connection = get_boto_connection(aws_access_key_id,
aws_secret_access_key,
bucketname=bucket)
try:
return connection.get_bucket(bucket)
except boto.exception.S3ResponseError:
return connection.get_bucket(bucket, validate=False)
def get_bucket(bucket, aws_access_key_id, aws_secret_access_key, **kwargs):
import boto
bucket = get_bucket_name(bucket)
if len(aws_access_key_id) > len(aws_secret_access_key):
logging.warn("The AWS credential keys aren't in the usual size,"
" are you using the correct ones?")
connection = boto.connect_s3(aws_access_key_id, aws_secret_access_key)
try:
return connection.get_bucket(bucket)
except boto.exception.S3ResponseError:
return connection.get_bucket(bucket, validate=False)
def get_bucket(bucket, aws_access_key_id, aws_secret_access_key, **kwargs):
import boto
bucket = get_bucket_name(bucket)
if len(aws_access_key_id) > len(aws_secret_access_key):
logging.warn("The AWS credential keys aren't in the usual size,"
" are you using the correct ones?")
connection = boto.connect_s3(aws_access_key_id, aws_secret_access_key)
try:
return connection.get_bucket(bucket)
except boto.exception.S3ResponseError:
return connection.get_bucket(bucket, validate=False)
def __init__(self, options, *args, **kwargs):
super(S3Writer, self).__init__(options, *args, **kwargs)
access_key = self.read_option('aws_access_key_id')
secret_key = self.read_option('aws_secret_access_key')
self.aws_region = self.read_option('aws_region')
bucket_name = get_bucket_name(self.read_option('bucket'))
self.logger.info('Starting S3Writer for bucket: %s' % bucket_name)
if self.aws_region is None:
self.aws_region = self._get_bucket_location(access_key, secret_key,
bucket_name)
self.conn = get_boto_connection(access_key, secret_key, self.aws_region,
bucket_name)
self.bucket = self.conn.get_bucket(bucket_name, validate=False)
self.save_metadata = self.read_option('save_metadata')
self.set_metadata('files_counter', Counter())
self.set_metadata('keys_written', [])
def __init__(self, options, *args, **kwargs):
super(S3Writer, self).__init__(options, *args, **kwargs)
access_key = self.read_option('aws_access_key_id')
secret_key = self.read_option('aws_secret_access_key')
self.aws_region = self.read_option('aws_region')
self.host = self.read_option('host')
bucket_name = get_bucket_name(self.read_option('bucket'))
self.logger.info('Starting S3Writer for bucket: %s' % bucket_name)
if self.aws_region is None:
self.aws_region = self._get_bucket_location(
access_key, secret_key, bucket_name)
self.conn = get_boto_connection(access_key, secret_key,
self.aws_region, bucket_name,
self.host)
self.bucket = self.conn.get_bucket(bucket_name, validate=False)
self.save_metadata = self.read_option('save_metadata')
self.set_metadata('files_counter', Counter())
self.set_metadata('keys_written', [])