def _get_bucket_location(self, access_key, secret_key, bucket):
try:
conn = get_boto_connection(access_key,
secret_key,
bucketname=bucket,
host=self.host)
return conn.get_bucket(
bucket).get_location() or DEFAULT_BUCKET_REGION
except:
return DEFAULT_BUCKET_REGION
def get_bucket(bucket, aws_access_key_id, aws_secret_access_key, **kwargs):
import boto
bucket = get_bucket_name(bucket)
connection = get_boto_connection(aws_access_key_id, aws_secret_access_key,
bucketname=bucket)
try:
return connection.get_bucket(bucket)
except boto.exception.S3ResponseError:
return connection.get_bucket(bucket, validate=False)
def get_bucket(bucket, aws_access_key_id, aws_secret_access_key, **kwargs):
import boto
bucket = get_bucket_name(bucket)
connection = get_boto_connection(aws_access_key_id,
aws_secret_access_key,
bucketname=bucket)
try:
return connection.get_bucket(bucket)
except boto.exception.S3ResponseError:
return connection.get_bucket(bucket, validate=False)
def execute(self):
writer_options = self.config.writer_options['options']
conn = get_boto_connection(
self.read_option('writer', 'aws_access_key_id'),
self.read_option('writer', 'aws_secret_access_key'),
self.read_option('writer', 'aws_region'),
self.read_option('writer', 'bucket'),
self.read_option('writer', 'host'))
self.dest_bucket = conn.get_bucket(self.read_option(
'writer', 'bucket'),
validate=False)
self.dest_filebase = self._get_filebase(writer_options)
super(S3Bypass, self).execute()
if writer_options.get('save_pointer'):
self._update_last_pointer(self.dest_bucket,
writer_options.get('save_pointer'),
writer_options.get('filebase'))
def __init__(self, options, *args, **kwargs):
super(S3Writer, self).__init__(options, *args, **kwargs)
access_key = self.read_option('aws_access_key_id')
secret_key = self.read_option('aws_secret_access_key')
self.aws_region = self.read_option('aws_region')
bucket_name = get_bucket_name(self.read_option('bucket'))
self.logger.info('Starting S3Writer for bucket: %s' % bucket_name)
if self.aws_region is None:
self.aws_region = self._get_bucket_location(access_key, secret_key,
bucket_name)
self.conn = get_boto_connection(access_key, secret_key, self.aws_region,
bucket_name)
self.bucket = self.conn.get_bucket(bucket_name, validate=False)
self.save_metadata = self.read_option('save_metadata')
self.set_metadata('files_counter', Counter())
self.set_metadata('keys_written', [])
def __init__(self, options, *args, **kwargs):
super(S3Writer, self).__init__(options, *args, **kwargs)
access_key = self.read_option('aws_access_key_id')
secret_key = self.read_option('aws_secret_access_key')
self.aws_region = self.read_option('aws_region')
self.host = self.read_option('host')
bucket_name = get_bucket_name(self.read_option('bucket'))
self.logger.info('Starting S3Writer for bucket: %s' % bucket_name)
if self.aws_region is None:
self.aws_region = self._get_bucket_location(
access_key, secret_key, bucket_name)
self.conn = get_boto_connection(access_key, secret_key,
self.aws_region, bucket_name,
self.host)
self.bucket = self.conn.get_bucket(bucket_name, validate=False)
self.save_metadata = self.read_option('save_metadata')
self.set_metadata('files_counter', Counter())
self.set_metadata('keys_written', [])
def _get_bucket_location(self, access_key, secret_key, bucket):
try:
conn = get_boto_connection(access_key, secret_key, bucketname=bucket)
return conn.get_bucket(bucket).get_location() or DEFAULT_BUCKET_REGION
except:
return DEFAULT_BUCKET_REGION
