def _get_bucket_location(self, access_key, secret_key, bucket): try: conn = get_boto_connection(access_key, secret_key, bucketname=bucket, host=self.host) return conn.get_bucket( bucket).get_location() or DEFAULT_BUCKET_REGION except: return DEFAULT_BUCKET_REGION
def get_bucket(bucket, aws_access_key_id, aws_secret_access_key, **kwargs): import boto bucket = get_bucket_name(bucket) connection = get_boto_connection(aws_access_key_id, aws_secret_access_key, bucketname=bucket) try: return connection.get_bucket(bucket) except boto.exception.S3ResponseError: return connection.get_bucket(bucket, validate=False)
def execute(self): writer_options = self.config.writer_options['options'] conn = get_boto_connection( self.read_option('writer', 'aws_access_key_id'), self.read_option('writer', 'aws_secret_access_key'), self.read_option('writer', 'aws_region'), self.read_option('writer', 'bucket'), self.read_option('writer', 'host')) self.dest_bucket = conn.get_bucket(self.read_option( 'writer', 'bucket'), validate=False) self.dest_filebase = self._get_filebase(writer_options) super(S3Bypass, self).execute() if writer_options.get('save_pointer'): self._update_last_pointer(self.dest_bucket, writer_options.get('save_pointer'), writer_options.get('filebase'))
def __init__(self, options, *args, **kwargs): super(S3Writer, self).__init__(options, *args, **kwargs) access_key = self.read_option('aws_access_key_id') secret_key = self.read_option('aws_secret_access_key') self.aws_region = self.read_option('aws_region') bucket_name = get_bucket_name(self.read_option('bucket')) self.logger.info('Starting S3Writer for bucket: %s' % bucket_name) if self.aws_region is None: self.aws_region = self._get_bucket_location(access_key, secret_key, bucket_name) self.conn = get_boto_connection(access_key, secret_key, self.aws_region, bucket_name) self.bucket = self.conn.get_bucket(bucket_name, validate=False) self.save_metadata = self.read_option('save_metadata') self.set_metadata('files_counter', Counter()) self.set_metadata('keys_written', [])
def __init__(self, options, *args, **kwargs): super(S3Writer, self).__init__(options, *args, **kwargs) access_key = self.read_option('aws_access_key_id') secret_key = self.read_option('aws_secret_access_key') self.aws_region = self.read_option('aws_region') self.host = self.read_option('host') bucket_name = get_bucket_name(self.read_option('bucket')) self.logger.info('Starting S3Writer for bucket: %s' % bucket_name) if self.aws_region is None: self.aws_region = self._get_bucket_location( access_key, secret_key, bucket_name) self.conn = get_boto_connection(access_key, secret_key, self.aws_region, bucket_name, self.host) self.bucket = self.conn.get_bucket(bucket_name, validate=False) self.save_metadata = self.read_option('save_metadata') self.set_metadata('files_counter', Counter()) self.set_metadata('keys_written', [])
def _get_bucket_location(self, access_key, secret_key, bucket): try: conn = get_boto_connection(access_key, secret_key, bucketname=bucket) return conn.get_bucket(bucket).get_location() or DEFAULT_BUCKET_REGION except: return DEFAULT_BUCKET_REGION