def test_has_s3_access(self):
# When RAZ is not enabled
assert_false(conf.has_s3_access(self.user))
# When only RAZ is enabled (S3 in Azure cluster)
reset = RAZ.IS_ENABLED.set_for_testing(True)
try:
assert_false(conf.has_s3_access(self.user))
finally:
reset()
conf.clear_cache()
# When RAZ is enabled along with S3 config
resets = [
RAZ.IS_ENABLED.set_for_testing(True),
conf.AWS_ACCOUNTS.set_for_testing({
'default': {
'region': 'us-west-2',
'host': 's3-us-west-2.amazonaws.com',
'allow_environment_credentials': 'false'
}
})
]
try:
assert_true(conf.has_s3_access(self.user))
finally:
for reset in resets:
reset()
conf.clear_cache()
def test_with_credentials(self):
try:
finish = conf.AWS_ACCOUNTS.set_for_testing({
'default': {
'access_key_id': 'access_key_id',
'secret_access_key': 'secret_access_key'
}
})
with patch('aws.client.conf_idbroker.get_conf') as get_conf:
with patch('aws.client.Client.get_s3_connection'):
get_conf.return_value = {}
client1 = get_client(name='default', fs='s3a')
client2 = get_client(name='default', fs='s3a', user='******')
provider = get_credential_provider('default', 'hue')
assert_equal(
provider.get_credentials().get('AccessKeyId'),
conf.AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get())
assert_equal(
client1, client2
) # Should be the same as no support for user based client with credentials & no Expiration
finally:
finish()
clear_cache()
conf.clear_cache()
def test_with_idbroker_on_ec2(self):
try:
finish = conf.AWS_ACCOUNTS.set_for_testing(
{}) # Set empty to test when no configs are set
with patch('aws.client.aws_conf.get_region') as get_region:
with patch('aws.client.conf_idbroker.get_conf') as get_conf:
with patch('aws.client.Client.get_s3_connection'):
with patch('aws.client.IDBroker.get_cab') as get_cab:
get_region.return_value = 'us-west-1'
get_conf.return_value = {
'fs.s3a.ext.cab.address': 'address'
}
get_cab.return_value = {
'Credentials': {
'AccessKeyId': 'AccessKeyId',
'Expiration': 0
}
}
client = Client.from_config(
None,
get_credential_provider('default', 'hue'))
assert_equal(
client._region, 'us-west-1'
) # Test different user have different clients
finally:
finish()
clear_cache()
conf.clear_cache()
def test_with_idbroker_and_config(self):
try:
finish = conf.AWS_ACCOUNTS.set_for_testing(
{'default': {
'region': 'ap-northeast-1'
}})
with patch('aws.client.conf_idbroker.get_conf') as get_conf:
with patch('aws.client.Client.get_s3_connection'):
with patch('aws.client.IDBroker.get_cab') as get_cab:
get_conf.return_value = {
'fs.s3a.ext.cab.address': 'address'
}
get_cab.return_value = {
'Credentials': {
'AccessKeyId': 'AccessKeyId',
'Expiration': 0
}
}
provider = get_credential_provider('default', 'hue')
assert_equal(
provider.get_credentials().get('AccessKeyId'),
'AccessKeyId')
client = Client.from_config(
conf.AWS_ACCOUNTS['default'],
get_credential_provider('default', 'hue'))
assert_equal(client._region, 'ap-northeast-1')
finally:
finish()
clear_cache()
conf.clear_cache()
def test_with_idbroker(self):
try:
finish = conf.AWS_ACCOUNTS.set_for_testing(
{}) # Set empty to test when no configs are set
with patch('aws.client.conf_idbroker.get_conf') as get_conf:
with patch('aws.client.Client.get_s3_connection'):
with patch('aws.client.IDBroker.get_cab') as get_cab:
get_conf.return_value = {
'fs.s3a.ext.cab.address': 'address'
}
get_cab.return_value = {
'Credentials': {
'AccessKeyId': 'AccessKeyId',
'Expiration': 0
}
}
provider = get_credential_provider('default', 'hue')
assert_equal(
provider.get_credentials().get('AccessKeyId'),
'AccessKeyId')
client1 = get_client(name='default',
fs='s3a',
user='******')
client2 = get_client(name='default',
fs='s3a',
user='******')
assert_not_equal(
client1, client2
) # Test that with Expiration 0 clients not equal
get_cab.return_value = {
'Credentials': {
'AccessKeyId':
'AccessKeyId',
'Expiration':
int(current_ms_from_utc()) + 10 * 1000
}
}
client3 = get_client(name='default',
fs='s3a',
user='******')
client4 = get_client(name='default',
fs='s3a',
user='******')
client5 = get_client(name='default',
fs='s3a',
user='******')
assert_equal(
client3, client4
) # Test that with 10 sec expiration, clients equal
assert_not_equal(
client4, client5
) # Test different user have different clients
finally:
finish()
clear_cache()
conf.clear_cache()
def test_with_raz_enabled(self):
with patch('aws.client.RazS3Connection') as raz_s3_connection:
resets = [
RAZ.IS_ENABLED.set_for_testing(True),
conf.AWS_ACCOUNTS.set_for_testing({
'default': {
'region': 'us-west-2',
'host': 's3-us-west-2.amazonaws.com',
'allow_environment_credentials': 'false'
}
})
]
try:
client = get_client(name='default', fs='s3a', user='******')
assert_true(client)
finally:
for reset in resets:
reset()
clear_cache()
conf.clear_cache()
def test_get_default_region():
# Verify that Hue can infer region from subdomain hosts
finish = conf.AWS_ACCOUNTS.set_for_testing(
{'default': {
'host': 's3.ap-northeast-2.amazonaws.com'
}})
try:
assert_equal('ap-northeast-2', get_default_region())
finally:
conf.clear_cache()
if finish:
finish()
# Verify that Hue can infer region from hyphenated hosts
finish = conf.AWS_ACCOUNTS.set_for_testing(
{'default': {
'host': 's3-ap-south-1.amazonaws.com'
}})
try:
assert_equal('ap-south-1', get_default_region())
finally:
conf.clear_cache()
if finish:
finish()
# Verify that Hue can infer region from hyphenated hosts
finish = conf.AWS_ACCOUNTS.set_for_testing(
{'default': {
'host': 's3.dualstack.ap-southeast-2.amazonaws.com'
}})
try:
assert_equal('ap-southeast-2', get_default_region())
finally:
conf.clear_cache()
if finish:
finish()
# Verify that Hue falls back to the default if the region is not valid
finish = conf.AWS_ACCOUNTS.set_for_testing(
{'default': {
'host': 's3-external-1.amazonaws.com'
}})
try:
assert_equal(Location.DEFAULT, get_default_region())
finally:
conf.clear_cache()
if finish:
finish()
# Verify that Hue uses the region if specified
finish = conf.AWS_ACCOUNTS.set_for_testing(
{'default': {
'host': '',
'region': 'ca-central-1'
}})
try:
assert_equal('ca-central-1', get_default_region())
finally:
conf.clear_cache()
if finish:
finish()
