def test_sigv4_only_region(tmpdir, monkeypatch):
monkeypatch.setenv('AWS_REGION', 'eu-central-1')
sigv4_check_apply()
bucket_name = bucket_name_mangle('sigv4')
creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'),
os.getenv('AWS_SECRET_ACCESS_KEY'))
cinfo = calling_format.from_store_name(bucket_name)
conn = cinfo.connect(creds)
try:
conn.create_bucket(bucket_name, location='eu-central-1')
except boto.exception.S3CreateError:
pass
source = unicode(tmpdir.join('source'))
contents = 'abcdefghijklmnopqrstuvwxyz\n' * 100
with open(source, 'wb') as f:
f.write(contents)
data_url = 's3://{0}/data'.format(bucket_name)
with open(source) as f:
uri_put_file(creds, data_url, f)
results = uri_get_file(creds, data_url)
assert contents == results
def test_sigv4_only_region(tmpdir, monkeypatch):
monkeypatch.setenv('AWS_REGION', 'eu-central-1')
sigv4_check_apply()
bucket_name = bucket_name_mangle('sigv4')
creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'),
os.getenv('AWS_SECRET_ACCESS_KEY'))
cinfo = calling_format.from_store_name(bucket_name)
conn = cinfo.connect(creds)
try:
conn.create_bucket(bucket_name, location='eu-central-1')
except boto.exception.S3CreateError:
pass
source = unicode(tmpdir.join('source'))
contents = 'abcdefghijklmnopqrstuvwxyz\n' * 100
with open(source, 'wb') as f:
f.write(contents)
data_url = 's3://{0}/data'.format(bucket_name)
with open(source) as f:
uri_put_file(creds, data_url, f)
results = uri_get_file(creds, data_url)
assert contents == results
def test_get_bucket_vs_certs():
"""Integration test for bucket naming issues."""
import boto.s3.connection
# Add dots to try to trip up TLS certificate validation.
bucket_name = bucket_name_mangle('wal-e.test.dots', delimiter='.')
with pytest.raises(boto.https_connection.InvalidCertificateException):
with FreshBucket(bucket_name, calling_format=SubdomainCallingFormat()):
pass
def test_404_termination(tmpdir):
bucket_name = bucket_name_mangle("wal-e-test-404-termination")
creds = Credentials(os.getenv("AWS_ACCESS_KEY_ID"), os.getenv("AWS_SECRET_ACCESS_KEY"))
with FreshBucket(bucket_name, host="s3.amazonaws.com", calling_format=OrdinaryCallingFormat()) as fb:
fb.create()
target = str(tmpdir.join("target"))
ret = do_lzop_get(creds, "s3://" + bucket_name + "/not-exist.lzo", target, False)
assert ret is False
def test_get_bucket_vs_certs():
"""Integration test for bucket naming issues."""
import boto.s3.connection
# Add dots to try to trip up TLS certificate validation.
bucket_name = bucket_name_mangle('wal-e.test.dots', delimiter='.')
with pytest.raises(boto.https_connection.InvalidCertificateException):
with FreshBucket(bucket_name, calling_format=SubdomainCallingFormat()):
pass
def test_sigv4_only_region(tmpdir, monkeypatch):
bucket_name = bucket_name_mangle('sigv4')
creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'),
os.getenv('AWS_SECRET_ACCESS_KEY'))
monkeypatch.setenv('AWS_REGION', 'eu-central-1')
def create_bucket_if_not_exists():
"""Create a bucket via path-based API calls.
This is because the preferred "$BUCKETNAME.s3.amazonaws"
subdomain doesn't yet exist for a non-existent bucket.
"""
monkeypatch.setenv('WALE_S3_ENDPOINT',
'https+path://s3-eu-central-1.amazonaws.com')
cinfo = calling_format.from_store_name(bucket_name)
conn = cinfo.connect(creds)
try:
conn.create_bucket(bucket_name, location='eu-central-1')
except boto.exception.S3CreateError:
pass
monkeypatch.delenv('WALE_S3_ENDPOINT')
create_bucket_if_not_exists()
def validate_bucket():
"""Validate the eu-central-1 bucket's existence
This is done using the subdomain that points to eu-central-1.
"""
sigv4_check_apply()
cinfo = calling_format.from_store_name(bucket_name)
conn = cinfo.connect(creds)
conn.get_bucket(bucket_name, validate=True)
validate_bucket()
def upload_download():
""" Test uri_put_file and uri_get_file in eu-central-1"""
source = str(tmpdir.join('source'))
contents = b'abcdefghijklmnopqrstuvwxyz\n' * 100
with open(source, 'wb') as f:
f.write(contents)
data_url = 's3://{0}/data'.format(bucket_name)
with open(source) as f:
uri_put_file(creds, data_url, f)
results = uri_get_file(creds, data_url)
assert contents == results
upload_download()
def test_sigv4_only_region(tmpdir, monkeypatch):
bucket_name = bucket_name_mangle("sigv4")
creds = Credentials(os.getenv("AWS_ACCESS_KEY_ID"), os.getenv("AWS_SECRET_ACCESS_KEY"))
monkeypatch.setenv("AWS_REGION", "eu-central-1")
def create_bucket_if_not_exists():
"""Create a bucket via path-based API calls.
This is because the preferred "$BUCKETNAME.s3.amazonaws"
subdomain doesn't yet exist for a non-existent bucket.
"""
monkeypatch.setenv("WALE_S3_ENDPOINT", "https+path://s3-eu-central-1.amazonaws.com")
cinfo = calling_format.from_store_name(bucket_name)
conn = cinfo.connect(creds)
try:
conn.create_bucket(bucket_name, location="eu-central-1")
except boto.exception.S3CreateError:
pass
monkeypatch.delenv("WALE_S3_ENDPOINT")
create_bucket_if_not_exists()
def validate_bucket():
"""Validate the eu-central-1 bucket's existence
This is done using the subdomain that points to eu-central-1.
"""
sigv4_check_apply()
cinfo = calling_format.from_store_name(bucket_name)
conn = cinfo.connect(creds)
conn.get_bucket(bucket_name, validate=True)
validate_bucket()
def upload_download():
""" Test uri_put_file and uri_get_file in eu-central-1"""
source = str(tmpdir.join("source"))
contents = b"abcdefghijklmnopqrstuvwxyz\n" * 100
with open(source, "wb") as f:
f.write(contents)
data_url = "s3://{0}/data".format(bucket_name)
with open(source) as f:
uri_put_file(creds, data_url, f)
results = uri_get_file(creds, data_url)
assert contents == results
upload_download()
def test_404_termination(tmpdir):
bucket_name = bucket_name_mangle('wal-e-test-404-termination')
creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'),
os.getenv('AWS_SECRET_ACCESS_KEY'))
with FreshBucket(bucket_name, host='s3.amazonaws.com',
calling_format=OrdinaryCallingFormat()) as fb:
fb.create()
target = unicode(tmpdir.join('target'))
ret = do_lzop_get(creds, 's3://' + bucket_name + '/not-exist.lzo',
target, False)
assert ret is False
def test_empty_latest_listing():
"""Test listing a 'backup-list LATEST' on an empty prefix."""
bucket_name = bucket_name_mangle('wal-e-test-empty-listing')
layout = storage.StorageLayout('s3://{0}/test-prefix'
.format(bucket_name))
with FreshBucket(bucket_name, host='s3.amazonaws.com',
calling_format=OrdinaryCallingFormat()) as fb:
fb.create()
bl = BackupList(fb.conn, layout, False)
found = list(bl.find_all('LATEST'))
assert len(found) == 0
def test_empty_latest_listing():
"""Test listing a 'backup-list LATEST' on an empty prefix."""
bucket_name = bucket_name_mangle('wal-e-test-empty-listing')
layout = storage.StorageLayout('s3://{0}/test-prefix'.format(bucket_name))
with FreshBucket(bucket_name,
host='s3.amazonaws.com',
calling_format=OrdinaryCallingFormat()) as fb:
fb.create()
bl = BackupList(fb.conn, layout, False)
found = list(bl.find_all('LATEST'))
assert len(found) == 0
def test_301_redirect():
"""Integration test for bucket naming issues this test."""
import boto.s3.connection
bucket_name = bucket_name_mangle('wal-e-test-301-redirect')
with pytest.raises(boto.exception.S3ResponseError) as e:
# Just initiating the bucket manipulation API calls is enough
# to provoke a 301 redirect.
with FreshBucket(bucket_name,
calling_format=OrdinaryCallingFormat()) as fb:
fb.create(location='us-west-1')
assert e.value.status == 301
def test_404_termination(tmpdir):
bucket_name = bucket_name_mangle('wal-e-test-404-termination')
creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'),
os.getenv('AWS_SECRET_ACCESS_KEY'))
with FreshBucket(bucket_name,
host='s3.amazonaws.com',
calling_format=OrdinaryCallingFormat()) as fb:
fb.create()
target = unicode(tmpdir.join('target'))
ret = do_lzop_get(creds, 's3://' + bucket_name + '/not-exist.lzo',
target, False)
assert ret is False
def test_301_redirect():
"""Integration test for bucket naming issues this test."""
import boto.s3.connection
bucket_name = bucket_name_mangle('wal-e-test-301-redirect')
with pytest.raises(boto.exception.S3ResponseError) as e:
# Just initiating the bucket manipulation API calls is enough
# to provoke a 301 redirect.
with FreshBucket(bucket_name,
calling_format=OrdinaryCallingFormat()) as fb:
fb.create(location='us-west-1')
assert e.value.status == 301
def test_subdomain_compatible():
"""Exercise a case where connecting is region-oblivious."""
creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'),
os.getenv('AWS_SECRET_ACCESS_KEY'))
bucket_name = bucket_name_mangle('wal-e-test-us-west-1-no-dots')
cinfo = calling_format.from_store_name(bucket_name)
with FreshBucket(bucket_name,
host='s3-us-west-1.amazonaws.com',
calling_format=connection.OrdinaryCallingFormat()) as fb:
fb.create(location='us-west-1')
conn = cinfo.connect(creds)
assert cinfo.region is None
assert cinfo.calling_format is connection.SubdomainCallingFormat
assert isinstance(conn.calling_format,
connection.SubdomainCallingFormat)
def test_subdomain_compatible():
"""Exercise a case where connecting is region-oblivious."""
creds = Credentials(os.getenv('AWS_ACCESS_KEY_ID'),
os.getenv('AWS_SECRET_ACCESS_KEY'))
bucket_name = bucket_name_mangle('wal-e-test-us-west-1-no-dots')
cinfo = calling_format.from_store_name(bucket_name)
with FreshBucket(bucket_name,
host='s3-us-west-1.amazonaws.com',
calling_format=connection.OrdinaryCallingFormat()) as fb:
fb.create(location='us-west-1')
conn = cinfo.connect(creds)
assert cinfo.region is None
assert cinfo.calling_format is connection.SubdomainCallingFormat
assert isinstance(conn.calling_format,
connection.SubdomainCallingFormat)
def test_backup_list(sts_conn, monkeypatch):
"""Test BackupList's compatibility with a test policy."""
monkeypatch.setenv('AWS_REGION', 'us-west-1')
bn = bucket_name_mangle('wal-e.sts.backup.list')
h = 's3-us-west-1.amazonaws.com'
cf = connection.OrdinaryCallingFormat()
fed = sts_conn.get_federation_token('wal-e-test-backup-list',
policy=make_policy(bn, 'test-prefix'))
layout = StorageLayout('s3://{0}/test-prefix'.format(bn))
creds = Credentials(fed.credentials.access_key, fed.credentials.secret_key,
fed.credentials.session_token)
with FreshBucket(bn, calling_format=cf, host=h) as fb:
fb.create(location='us-west-1')
cinfo = calling_format.from_store_name(bn)
conn = cinfo.connect(creds)
conn.host = h
backups = list(BackupList(conn, layout, True))
assert not backups
def test_uri_put_file(sts_conn, monkeypatch):
monkeypatch.setenv('AWS_REGION', 'us-west-1')
bn = bucket_name_mangle('wal-e.sts.uri.put.file')
cf = connection.OrdinaryCallingFormat()
policy_text = make_policy(bn, 'test-prefix', allow_get_location=True)
fed = sts_conn.get_federation_token('wal-e-test-uri-put-file',
policy=policy_text)
key_path = 'test-prefix/test-key'
creds = Credentials(fed.credentials.access_key,
fed.credentials.secret_key,
fed.credentials.session_token)
with FreshBucket(bn, keys=[key_path], calling_format=cf,
host='s3-us-west-1.amazonaws.com') as fb:
fb.create(location='us-west-1')
uri_put_file(creds, 's3://' + bn + '/' + key_path,
StringIO('test-content'))
k = connection.Key(fb.conn.get_bucket(bn, validate=False))
k.name = key_path
assert k.get_contents_as_string() == b'test-content'
def test_backup_list(sts_conn, monkeypatch):
"""Test BackupList's compatibility with a test policy."""
monkeypatch.setenv('AWS_REGION', 'us-west-1')
bn = bucket_name_mangle('wal-e.sts.backup.list')
h = 's3-us-west-1.amazonaws.com'
cf = connection.OrdinaryCallingFormat()
fed = sts_conn.get_federation_token('wal-e-test-backup-list',
policy=make_policy(bn, 'test-prefix'))
layout = StorageLayout('s3://{0}/test-prefix'.format(bn))
creds = Credentials(fed.credentials.access_key,
fed.credentials.secret_key,
fed.credentials.session_token)
with FreshBucket(bn, calling_format=cf, host=h) as fb:
fb.create(location='us-west-1')
cinfo = calling_format.from_store_name(bn)
conn = cinfo.connect(creds)
conn.host = h
backups = list(BackupList(conn, layout, True))
assert not backups
def test_uri_put_file(sts_conn, monkeypatch):
monkeypatch.setenv('AWS_REGION', 'us-west-1')
bn = bucket_name_mangle('wal-e.sts.uri.put.file')
cf = connection.OrdinaryCallingFormat()
policy_text = make_policy(bn, 'test-prefix', allow_get_location=True)
fed = sts_conn.get_federation_token('wal-e-test-uri-put-file',
policy=policy_text)
key_path = 'test-prefix/test-key'
creds = Credentials(fed.credentials.access_key, fed.credentials.secret_key,
fed.credentials.session_token)
with FreshBucket(bn,
keys=[key_path],
calling_format=cf,
host='s3-us-west-1.amazonaws.com') as fb:
fb.create(location='us-west-1')
uri_put_file(creds, 's3://' + bn + '/' + key_path,
StringIO('test-content'))
k = connection.Key(fb.conn.get_bucket(bn, validate=False))
k.name = key_path
assert k.get_contents_as_string() == b'test-content'
def test_policy(sts_conn, monkeypatch):
"""Sanity checks for the intended ACLs of the policy"""
monkeypatch.setenv('AWS_REGION', 'us-west-1')
# Use periods to force OrdinaryCallingFormat when using
# calling_format.from_store_name.
bn = bucket_name_mangle('wal-e.sts.list.test')
h = 's3-us-west-1.amazonaws.com'
cf = connection.OrdinaryCallingFormat()
fed = sts_conn.get_federation_token('wal-e-test-list-bucket',
policy=make_policy(bn, 'test-prefix'))
test_payload = 'wal-e test'
keys = [
'test-prefix/hello', 'test-prefix/world', 'not-in-prefix/goodbye',
'not-in-prefix/world'
]
creds = Credentials(fed.credentials.access_key, fed.credentials.secret_key,
fed.credentials.session_token)
with FreshBucket(bn, keys=keys, calling_format=cf, host=h) as fb:
# Superuser creds, for testing keys not in the prefix.
bucket_superset_creds = fb.create(location='us-west-1')
cinfo = calling_format.from_store_name(bn)
conn = cinfo.connect(creds)
conn.host = h
# Bucket using the token, subject to the policy.
bucket = conn.get_bucket(bn, validate=False)
for name in keys:
if name.startswith('test-prefix/'):
# Test the PUT privilege.
k = connection.Key(bucket)
else:
# Not in the prefix, so PUT will not work.
k = connection.Key(bucket_superset_creds)
k.key = name
k.set_contents_from_string(test_payload)
# Test listing keys within the prefix.
prefix_fetched_keys = list(bucket.list(prefix='test-prefix/'))
assert len(prefix_fetched_keys) == 2
# Test the GET privilege.
for key in prefix_fetched_keys:
assert key.get_contents_as_string() == b'wal-e test'
# Try a bogus listing outside the valid prefix.
with pytest.raises(exception.S3ResponseError) as e:
list(bucket.list(prefix=''))
assert e.value.status == 403
# Test the rejection of PUT outside of prefix.
k = connection.Key(bucket)
k.key = 'not-in-prefix/world'
with pytest.raises(exception.S3ResponseError) as e:
k.set_contents_from_string(test_payload)
assert e.value.status == 403
def test_policy(sts_conn, monkeypatch):
"""Sanity checks for the intended ACLs of the policy"""
monkeypatch.setenv('AWS_REGION', 'us-west-1')
# Use periods to force OrdinaryCallingFormat when using
# calling_format.from_store_name.
bn = bucket_name_mangle('wal-e.sts.list.test')
h = 's3-us-west-1.amazonaws.com'
cf = connection.OrdinaryCallingFormat()
fed = sts_conn.get_federation_token('wal-e-test-list-bucket',
policy=make_policy(bn, 'test-prefix'))
test_payload = 'wal-e test'
keys = ['test-prefix/hello', 'test-prefix/world',
'not-in-prefix/goodbye', 'not-in-prefix/world']
creds = Credentials(fed.credentials.access_key,
fed.credentials.secret_key,
fed.credentials.session_token)
with FreshBucket(bn, keys=keys, calling_format=cf, host=h) as fb:
# Superuser creds, for testing keys not in the prefix.
bucket_superset_creds = fb.create(location='us-west-1')
cinfo = calling_format.from_store_name(bn)
conn = cinfo.connect(creds)
conn.host = h
# Bucket using the token, subject to the policy.
bucket = conn.get_bucket(bn, validate=False)
for name in keys:
if name.startswith('test-prefix/'):
# Test the PUT privilege.
k = connection.Key(bucket)
else:
# Not in the prefix, so PUT will not work.
k = connection.Key(bucket_superset_creds)
k.key = name
k.set_contents_from_string(test_payload)
# Test listing keys within the prefix.
prefix_fetched_keys = list(bucket.list(prefix='test-prefix/'))
assert len(prefix_fetched_keys) == 2
# Test the GET privilege.
for key in prefix_fetched_keys:
assert key.get_contents_as_string() == b'wal-e test'
# Try a bogus listing outside the valid prefix.
with pytest.raises(exception.S3ResponseError) as e:
list(bucket.list(prefix=''))
assert e.value.status == 403
# Test the rejection of PUT outside of prefix.
k = connection.Key(bucket)
k.key = 'not-in-prefix/world'
with pytest.raises(exception.S3ResponseError) as e:
k.set_contents_from_string(test_payload)
assert e.value.status == 403
