def setUp(self): """Setup users, projects, and start a test server.""" super(S3APITestCase, self).setUp() self.flags(auth_driver='nova.auth.ldapdriver.FakeLdapDriver', buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'), s3_host='127.0.0.1') self.auth_manager = manager.AuthManager() self.admin_user = self.auth_manager.create_user('admin', admin=True) self.admin_project = self.auth_manager.create_project('admin', self.admin_user) shutil.rmtree(FLAGS.buckets_path) os.mkdir(FLAGS.buckets_path) router = s3server.S3Application(FLAGS.buckets_path) server = wsgi.Server() server.start(router, FLAGS.s3_port, host=FLAGS.s3_host) if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.set('Boto', 'num_retries', '0') conn = s3.S3Connection(aws_access_key_id=self.admin_user.access, aws_secret_access_key=self.admin_user.secret, host=FLAGS.s3_host, port=FLAGS.s3_port, is_secure=False, calling_format=s3.OrdinaryCallingFormat()) self.conn = conn def get_http_connection(host, is_secure): """Get a new S3 connection, don't attempt to reuse connections.""" return self.conn.new_http_connection(host, is_secure) self.conn.get_http_connection = get_http_connection
def __init__(self, config): """ Creation of the manager. Since it is only s3 compatible we need to specify the endpoint in the config """ super(CleversafeClient, self).__init__(__name__) self._config = config self._host = config["host"] self._public_host = config["public_host"] self._access_key = config["aws_access_key_id"] self._secret_key = config["aws_secret_access_key"] self._username = config["username"] self._password = config["password"] self._permissions_order = { "read-storage": 1, "write-storage": 2, "admin-storage": 3, "disabled": 0, } self._permissions_value = ["disabled", "readOnly", "readWrite", "owner"] self._auth = requests.auth.HTTPBasicAuth(self._username, self._password) self._conn = connect_s3( aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, host=self._public_host, calling_format=connection.OrdinaryCallingFormat(), ) self._bucket_name_id_table = {} self._update_bucket_name_id_table() self._user_name_id_table = {} self._user_id_name_table = {} self._update_user_name_id_table()
def setUp(self): """Setup users, projects, and start a test server.""" super(S3APITestCase, self).setUp() tempdir = self.useFixture(fixtures.TempDir()) conf = self.useFixture(config_fixture.Config()) conf.config(buckets_path=tempdir.path, s3_listen='127.0.0.1', s3_listen_port=0) self.server = s3server.get_wsgi_server() # NOTE(ft): this requires eventlet.monkey_patch, which is called in # tests/unit/__init__.py. Remove it out from there if you get these # tests rid of server run self.server.start() self.addCleanup(self.server.stop) if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.set('Boto', 'num_retries', '0') conn = s3.S3Connection(aws_access_key_id='fake', aws_secret_access_key='fake', host=CONF.s3_listen, port=self.server.port, is_secure=False, calling_format=s3.OrdinaryCallingFormat()) self.conn = conn def get_http_connection(*args): """Get a new S3 connection, don't attempt to reuse connections.""" return self.conn.new_http_connection(*args) self.conn.get_http_connection = get_http_connection
def __init__(self, username, access_key, secret_key, rate_limit=None, host=None): self.username = username self.access_key = access_key self.secret_key = secret_key self.rate_limit = rate_limit self.rate_limiter = RateLimiter(self.rate_limit) self.callbacks = CallbackAggregator() self.multipart_status_callbacks = CallbackAggregator() self.host = host self.logger = logging.getLogger(__name__) if self.rate_limit: self.callbacks.add_callback(self.rate_limiter) if self.host =='s3.amazonaws.com': self.connection = boto.connect_s3(self.access_key, self.secret_key) else: self.connection = boto.connect_s3(aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, is_secure=False, host=self.host, port=8773, calling_format=boto_s3_connection_class.OrdinaryCallingFormat(), path="/services/Walrus") if not self.connection: raise S3AuthError("check access_key and secret_key") self.bucket = self.connection.lookup(username) if not self.bucket: raise S3AuthError("check access_key and secret_key")
def setUp(self): """Setup users, projects, and start a test server.""" super(S3APITestCase, self).setUp() self.flags(buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'), s3_host='127.0.0.1') shutil.rmtree(CONF.buckets_path) os.mkdir(CONF.buckets_path) router = s3server.S3Application(CONF.buckets_path) self.server = wsgi.Server("S3 Objectstore", router, host=CONF.s3_host, port=0) self.server.start() if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.set('Boto', 'num_retries', '0') conn = s3.S3Connection(aws_access_key_id='fake', aws_secret_access_key='fake', host=CONF.s3_host, port=self.server.port, is_secure=False, calling_format=s3.OrdinaryCallingFormat()) self.conn = conn def get_http_connection(*args): """Get a new S3 connection, don't attempt to reuse connections.""" return self.conn.new_http_connection(*args) self.conn.get_http_connection = get_http_connection
def _create_s3_connection(config, region): calling_format = bsc.OrdinaryCallingFormat() if region: conn = bs.connect_to_region( region, aws_access_key_id=config[tac.key_id], aws_secret_access_key=config[tac.secret_key], proxy=config.get(tac.proxy_hostname), proxy_port=config.get(tac.proxy_port), proxy_user=config.get(tac.proxy_username), proxy_pass=config.get(tac.proxy_password), is_secure=True, validate_certs=True, calling_format=calling_format) else: if (not os.environ.get("S3_USE_SIGV4") and not config.get(asc.bucket_name)): calling_format = bsc.SubdomainCallingFormat() conn = boto.connect_s3( host=config[asc.host_name], aws_access_key_id=config[tac.key_id], aws_secret_access_key=config[tac.secret_key], proxy=config.get(tac.proxy_hostname), proxy_port=config.get(tac.proxy_port), proxy_user=config.get(tac.proxy_username), proxy_pass=config.get(tac.proxy_password), is_secure=True, validate_certs=True, calling_format=calling_format) return conn
def __init__(self, config, lazy=False, host_aliases={}): """ Config map should be a map from hostname to args, e.g.: { "s3.amazonaws.com": { "aws_access_key_id": "foo", "aws_secret_access_key": "bar", "is_secure": False, . . . }, } :param host_aliases: A *REGEX* map from names that match the regex to hostnames provided in config e.g. ``{'aws\.accesssor1\.mirror': 's3.amazonaws.com'}`` """ self.config = config for host, kwargs in self.config.iteritems(): # we need to pass the host argument in when we connect, so # set it here kwargs["host"] = host if 'calling_format' not in kwargs: kwargs["calling_format"] = connection.OrdinaryCallingFormat() self.host_aliases = host_aliases self.conns = {} if not lazy: self.connect()
def _s3connection_opts_from_uri(impl): # 'impl' should look like: # # <protocol>+<calling_format>://[user:pass]@<host>[:port] # # A concrete example: # # https+virtualhost://user:pass@localhost:1235 o = urlparse.urlparse(impl, allow_fragments=False) if o.scheme is not None: proto_match = re.match( r'(?P<protocol>http|https)\+' r'(?P<format>virtualhost|path|subdomain)', o.scheme) if proto_match is None: raise UserException( msg='WALE_S3_ENDPOINT URI scheme is invalid', detail='The scheme defined is ' + repr(o.scheme), hint='An example of a valid scheme is https+virtualhost.') opts = {} if proto_match.group('protocol') == 'http': opts['is_secure'] = False else: # Constrained by prior regexp. proto_match.group('protocol') == 'https' opts['is_secure'] = True f = proto_match.group('format') if f == 'virtualhost': opts['calling_format'] = connection.VHostCallingFormat() elif f == 'path': opts['calling_format'] = connection.OrdinaryCallingFormat() elif f == 'subdomain': opts['calling_format'] = connection.SubdomainCallingFormat() else: # Constrained by prior regexp. assert False if o.username is not None or o.password is not None: raise UserException( msg='WALE_S3_ENDPOINT does not support username or password') if o.hostname is not None: opts['host'] = o.hostname if o.port is not None: opts['port'] = o.port if o.path: raise UserException( msg='WALE_S3_ENDPOINT does not support a URI path', detail='Path is {0!r}'.format(o.path)) if o.query: raise UserException( msg='WALE_S3_ENDPOINT does not support query parameters') return opts
def connection(self): if self._connection: return self._connection self._connection = boto.connect_s3( aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=self.s3_connection_host, calling_format=boto_s3_connection.OrdinaryCallingFormat()) return self._connection
def bucket(self): boto_connection = boto.connect_s3( self.config.access_key, self.config.access_secret, calling_format=connection.OrdinaryCallingFormat()) return boto_connection.get_bucket(self.config.bucket) try: return boto_connection.get_bucket(self.config.bucket) except boto.exception.S3ResponseError as e: if e.status == 404: logging.info('Creating bucket: {}'.format(self.config.bucket)) return boto_connection.create_bucket(self.config.bucket) raise
def bucket(self): if self.use_interoperable_auth: gs_connection = boto.connect_gs( self.config.access_key, self.config.access_secret, calling_format=connection.OrdinaryCallingFormat()) # Always use our internal cacerts.txt file. This fixes an issue with the # PyInstaller-based frozen distribution, while allowing us to continue to # verify certificates and use a secure connection. gs_connection.ca_certificates_file = _certs_path else: gs_connection = storage.get_connection( self.config.project, self.config.email, self.config.key_path) return gs_connection.get_bucket(self.config.bucket)
class EucalyptusClient(ec2client.EC2Client): cloudType = 'eucalyptus' # XXX will need their own image type RBUILDER_BUILD_TYPE = 'RAW_FS_IMAGE' ImagePrefix = 'emi-' CallingFormat = s3connection.OrdinaryCallingFormat() configurationDescriptorXmlData = _configurationDescriptorXmlData credentialsDescriptorXmlData = _credentialsDescriptorXmlData PermittedS3Users = None class Cloud(ec2client.EC2_Cloud): pass class Image(ec2client.EC2_Image): pass class Instance(ec2client.EC2_Instance): pass _configNameMap = [] class _ImageMap(ec2client.EC2Client._ImageMap): def __init__(self, imageList): ec2client.EC2Client._ImageMap.__init__(self, imageList) for img in imageList: # Hash images by target image id too if img._targetImageId is not None: self._ids[img._targetImageId] = img def drvCreateCloud(self, descriptorData): return ec2client.baseDriver.BaseDriver.drvCreateCloud(self, descriptorData) def drvVerifyCloudConfiguration(self, config): certificateData = config.get('certificateData') certificateKeyData = config.get('certificateKeyData') config.update(certificateData=certificateData, certificateKeyData=certificateKeyData) config.update((k, self._strip(v)) for k, v in config.items()) # Seed the target configuration self._targetConfig = config # Validate credentials cli = self.drvCreateCloudClient(config) # Do a call to force cred validation try: cli.get_all_regions() except ec2client.EC2ResponseError, e: raise errors.ResponseError(e.status, self._getErrorMessage(e), e.body) self._targetConfig = None
def bucket(self): if self.config.oauth2: enable_oauth2_auth_handler() gs_connection = boto.connect_gs( self.config.access_key, self.config.access_secret, calling_format=connection.OrdinaryCallingFormat()) # Always use our internal cacerts.txt file. This fixes an issue with the # PyInstaller-based frozen distribution, while allowing us to continue to # verify certificates and use a secure connection. gs_connection.ca_certificates_file = utils.get_cacerts_path() try: return gs_connection.get_bucket(self.config.bucket) except boto.exception.GSResponseError as e: if e.status == 404: logging.info('Creating bucket: {}'.format(self.config.bucket)) return gs_connection.create_bucket(self.config.bucket) raise
def test_classic_get_location(): """Exercise get location on a s3-classic bucket.""" creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'), os.getenv('AWS_SECRET_ACCESS_KEY')) bucket_name = 'wal-e-test.classic.get.location' cinfo = calling_format.from_store_name(bucket_name) with FreshBucket(bucket_name, host='s3.amazonaws.com', calling_format=connection.OrdinaryCallingFormat()) as fb: fb.create() conn = cinfo.connect(creds) assert cinfo.region == 'us-standard' assert cinfo.calling_format is connection.OrdinaryCallingFormat assert conn.host == 's3.amazonaws.com'
def connect(self): try: self._client = boto.connect_s3( aws_access_key_id=self.config["access_key"], aws_secret_access_key=self.config["secret_key"], host=self.config["host"], is_secure=self.config["is_secure"], port=self.config["port"], calling_format=s3_connection.OrdinaryCallingFormat(), ) self._bucket = self._client.get_bucket(self.config["bucket"]) except Exception: log("CephAdapter connect error", params={log_const.KEY_NAME: log_const.HANDLED_EXCEPTION_VALUE}, level="ERROR", exc_info=True) monitoring.got_counter("ceph_connection_exception") raise
def test_get_location_errors(monkeypatch): """Simulate situations where get_location fails Exercise both the case where IAM refuses the privilege to get the bucket location and where some other S3ResponseError is raised instead. """ bucket_name = 'wal-e.test.403.get.location' def just_403(self): raise boto.exception.S3ResponseError(status=403, reason=None, body=None) def unhandled_404(self): raise boto.exception.S3ResponseError(status=404, reason=None, body=None) creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'), os.getenv('AWS_SECRET_ACCESS_KEY')) with FreshBucket(bucket_name, calling_format=connection.OrdinaryCallingFormat()): cinfo = calling_format.from_store_name(bucket_name) # Provoke a 403 when trying to get the bucket location. monkeypatch.setattr(boto.s3.bucket.Bucket, 'get_location', just_403) cinfo.connect(creds) assert cinfo.region == 'us-standard' assert cinfo.calling_format is connection.OrdinaryCallingFormat cinfo = calling_format.from_store_name(bucket_name) # Provoke an unhandled S3ResponseError, in this case 404 not # found. monkeypatch.setattr(boto.s3.bucket.Bucket, 'get_location', unhandled_404) with pytest.raises(boto.exception.S3ResponseError) as e: cinfo.connect(creds) assert e.value.status == 404
def test_classic_get_location(): """Exercise get location on a s3-classic bucket.""" aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID') aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY') bucket_name = ('wal-e-test.classic.get.location.' + aws_access_key_id.lower()) cinfo = calling_format.from_bucket_name(bucket_name) with FreshBucket(bucket_name, host='s3.amazonaws.com', calling_format=connection.OrdinaryCallingFormat()) as fb: fb.create() conn = cinfo.connect(aws_access_key_id, aws_secret_access_key) assert cinfo.region == 'us-standard' assert cinfo.calling_format is connection.OrdinaryCallingFormat assert conn.host == 's3.amazonaws.com'
def test_subdomain_compatible(): """Exercise a case where connecting is region-oblivious.""" creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'), os.getenv('AWS_SECRET_ACCESS_KEY')) bucket_name = 'wal-e-test-us-west-1-no-dots' cinfo = calling_format.from_store_name(bucket_name) with FreshBucket(bucket_name, host='s3-us-west-1.amazonaws.com', calling_format=connection.OrdinaryCallingFormat()) as fb: fb.create(location='us-west-1') conn = cinfo.connect(creds) assert cinfo.region is None assert cinfo.calling_format is connection.SubdomainCallingFormat assert isinstance(conn.calling_format, connection.SubdomainCallingFormat)
def create_bucket(self, bucket_name, access_key=None, secret_key=None): """ Requires a default template created on cleversafe """ if not access_key: access_key = self._access_key if not secret_key: secret_key = self._secret_key creds = {"host": self._public_host} creds["aws_access_key_id"] = access_key creds["aws_secret_access_key"] = secret_key conn = connect_s3(calling_format=connection.OrdinaryCallingFormat(), **creds) try: bucket = conn.create_bucket(bucket_name) self._update_bucket_name_id_table() return bucket except S3ResponseError as exce: msg = "Create bucket failed with error code: {0}" self.logger.error(msg.format(exce.error_code)) raise RequestError(str(exce), exce.error_code)
def test_backup_list(sts_conn): """Test BackupList's compatibility with a test policy.""" bn = 'wal-e.sts.backup.list' h = 's3-us-west-1.amazonaws.com' cf = connection.OrdinaryCallingFormat() fed = sts_conn.get_federation_token('wal-e-test-backup-list', policy=make_policy(bn, 'test-prefix')) layout = StorageLayout('s3://{0}/test-prefix'.format(bn)) creds = Credentials(fed.credentials.access_key, fed.credentials.secret_key, fed.credentials.session_token) with FreshBucket(bn, calling_format=cf, host=h) as fb: fb.create(location='us-west-1') cinfo = calling_format.from_store_name(bn) conn = cinfo.connect(creds) conn.host = h backups = list(BackupList(conn, layout, True)) assert not backups
def test_subdomain_compatible(): """Exercise a case where connecting is region-oblivious.""" aws_access_key = os.getenv('AWS_ACCESS_KEY_ID') bucket_name = 'wal-e-test-us-west-1-no-dots' + aws_access_key.lower() cinfo = calling_format.from_bucket_name(bucket_name) aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID') aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY') with FreshBucket(bucket_name, host='s3-us-west-1.amazonaws.com', calling_format=connection.OrdinaryCallingFormat()) as fb: fb.create(location='us-west-1') conn = cinfo.connect(aws_access_key_id, aws_secret_access_key) assert cinfo.region is None assert cinfo.calling_format is connection.SubdomainCallingFormat assert isinstance(conn.calling_format, connection.SubdomainCallingFormat)
def test_uri_put_file(sts_conn): bn = 'wal-e.sts.uri.put.file' cf = connection.OrdinaryCallingFormat() policy_text = make_policy(bn, 'test-prefix', allow_get_location=True) fed = sts_conn.get_federation_token('wal-e-test-uri-put-file', policy=policy_text) key_path = 'test-prefix/test-key' creds = Credentials(fed.credentials.access_key, fed.credentials.secret_key, fed.credentials.session_token) with FreshBucket(bn, keys=[key_path], calling_format=cf, host='s3-us-west-1.amazonaws.com') as fb: fb.create(location='us-west-1') uri_put_file(creds, 's3://' + bn + '/' + key_path, StringIO('test-content')) k = connection.Key(fb.conn.get_bucket(bn, validate=False)) k.name = key_path assert k.get_contents_as_string() == 'test-content'
def test_real_get_location(): """Exercise a case where a get location call is needed. In cases where a bucket has offensive characters -- like dots -- that would otherwise break TLS, test sniffing the right endpoint so it can be used to address the bucket. """ creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'), os.getenv('AWS_SECRET_ACCESS_KEY')) bucket_name = 'wal-e-test-us-west-1.get.location' cinfo = calling_format.from_store_name(bucket_name) with FreshBucket(bucket_name, host='s3-us-west-1.amazonaws.com', calling_format=connection.OrdinaryCallingFormat()) as fb: fb.create(location='us-west-1') conn = cinfo.connect(creds) assert cinfo.region == 'us-west-1' assert cinfo.calling_format is connection.OrdinaryCallingFormat assert conn.host == 's3-us-west-1.amazonaws.com'
def bucket(self): boto_connection = boto.connect_s3( self.config.access_key, self.config.access_secret, calling_format=connection.OrdinaryCallingFormat()) return boto_connection.get_bucket(self.config.bucket)
parser.add_argument('--host', type=str, default='10.5.100.1', required=False) parser.add_argument('-n', '--num-objs', type=int, default=1, required=False) parser.add_argument('--bytes', type=int, default=1024 * 110, required=False) parser.add_argument('--objnamelen', type=int, default=0, required=False) parser.add_argument('--bucket', type=str, default='testbucket', required=False) args = parser.parse_args() print "Using S3 endpoint '%s:%s'" % (args.host, args.port) access_key, secret_key = get_ec2_creds() conn = boto.connect_s3( aws_access_key_id=access_key, aws_secret_access_key=secret_key, host=args.host, port=args.port, is_secure=False, calling_format=connection.OrdinaryCallingFormat(), ) bname = args.bucket all_buckets = conn.get_all_buckets() bucket = [b for b in all_buckets if b.name == bname] if bucket: bucket = bucket[0] num_objs = len(list(bucket.list())) print('Bucket {} already exists and contains {} objects'.format( bucket.name, num_objs)) else: print 'Creating new bucket {}'.format(bname) bucket = conn.create_bucket(bname) k = key.Key(bucket)
def test_policy(sts_conn, monkeypatch): """Sanity checks for the intended ACLs of the policy""" monkeypatch.setenv('AWS_REGION', 'us-west-1') # Use periods to force OrdinaryCallingFormat when using # calling_format.from_store_name. bn = bucket_name_mangle('wal-e.sts.list.test') h = 's3-us-west-1.amazonaws.com' cf = connection.OrdinaryCallingFormat() fed = sts_conn.get_federation_token('wal-e-test-list-bucket', policy=make_policy(bn, 'test-prefix')) test_payload = 'wal-e test' keys = [ 'test-prefix/hello', 'test-prefix/world', 'not-in-prefix/goodbye', 'not-in-prefix/world' ] creds = Credentials(fed.credentials.access_key, fed.credentials.secret_key, fed.credentials.session_token) with FreshBucket(bn, keys=keys, calling_format=cf, host=h) as fb: # Superuser creds, for testing keys not in the prefix. bucket_superset_creds = fb.create(location='us-west-1') cinfo = calling_format.from_store_name(bn) conn = cinfo.connect(creds) conn.host = h # Bucket using the token, subject to the policy. bucket = conn.get_bucket(bn, validate=False) for name in keys: if name.startswith('test-prefix/'): # Test the PUT privilege. k = connection.Key(bucket) else: # Not in the prefix, so PUT will not work. k = connection.Key(bucket_superset_creds) k.key = name k.set_contents_from_string(test_payload) # Test listing keys within the prefix. prefix_fetched_keys = list(bucket.list(prefix='test-prefix/')) assert len(prefix_fetched_keys) == 2 # Test the GET privilege. for key in prefix_fetched_keys: assert key.get_contents_as_string() == b'wal-e test' # Try a bogus listing outside the valid prefix. with pytest.raises(exception.S3ResponseError) as e: list(bucket.list(prefix='')) assert e.value.status == 403 # Test the rejection of PUT outside of prefix. k = connection.Key(bucket) k.key = 'not-in-prefix/world' with pytest.raises(exception.S3ResponseError) as e: k.set_contents_from_string(test_payload) assert e.value.status == 403
def __init__(self, config={}, lazy=False, host_aliases={}, stream_status=False): """ Config map should be a map from hostname to args, e.g.: { "cleversafe.service.consul: { "aws_access_key_id": "foo", "aws_secret_access_key": "bar", "is_secure": False, . . . }, } :param host_aliases: A *REGEX* map from names that match the regex to hostnames provided in config e.g. ``{'aws\.accessor1\.mirror': 'cleversafe.service.consul'}`` """ self.config = config for host, kwargs in iteritems(self.config): # we need to pass the host argument in when we connect, so # set it here kwargs["host"] = host if 'calling_format' not in kwargs: kwargs["calling_format"] = connection.OrdinaryCallingFormat() self.host_aliases = host_aliases self.conns = {} if not lazy: self.connect() self.s3_inst_info = { 'ceph': { 'secure': True, 'url': 'ceph.service.consul', 'access_key': "", 'secret_key': "" }, 'ceph2': { 'secure': True, 'url': 'gdc-cephb-objstore.osdc.io', 'access_key': "", 'secret_key': "" }, 'cleversafe': { 'secure': True, 'url': 'gdc-accessors.osdc.io', 'access_key': "", 'secret_key': "" } } self.stream_status = stream_status # magic number here for multipart chunk size, change with care self.mp_chunk_size = 1073741824 # 1GiB # semi-magic number here, worth playing with if speed issues seen self.chunk_size = 16777216