def test_with_credentials(self):
try:
finish = conf.AWS_ACCOUNTS.set_for_testing({
'default': {
'access_key_id': 'access_key_id',
'secret_access_key': 'secret_access_key'
}
})
with patch('aws.client.conf_idbroker.get_conf') as get_conf:
with patch('aws.client.Client.get_s3_connection'):
get_conf.return_value = {}
client1 = get_client(name='default', fs='s3a')
client2 = get_client(name='default', fs='s3a', user='******')
provider = get_credential_provider('default', 'hue')
assert_equal(
provider.get_credentials().get('AccessKeyId'),
conf.AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get())
assert_equal(
client1, client2
) # Should be the same as no support for user based client with credentials & no Expiration
finally:
finish()
clear_cache()
conf.clear_cache()
def test_with_idbroker_on_ec2(self):
try:
finish = conf.AWS_ACCOUNTS.set_for_testing(
{}) # Set empty to test when no configs are set
with patch('aws.client.aws_conf.get_region') as get_region:
with patch('aws.client.conf_idbroker.get_conf') as get_conf:
with patch('aws.client.Client.get_s3_connection'):
with patch('aws.client.IDBroker.get_cab') as get_cab:
get_region.return_value = 'us-west-1'
get_conf.return_value = {
'fs.s3a.ext.cab.address': 'address'
}
get_cab.return_value = {
'Credentials': {
'AccessKeyId': 'AccessKeyId',
'Expiration': 0
}
}
client = Client.from_config(
None,
get_credential_provider('default', 'hue'))
assert_equal(
client._region, 'us-west-1'
) # Test different user have different clients
finally:
finish()
clear_cache()
conf.clear_cache()
def test_with_idbroker_and_config(self):
try:
finish = conf.AWS_ACCOUNTS.set_for_testing(
{'default': {
'region': 'ap-northeast-1'
}})
with patch('aws.client.conf_idbroker.get_conf') as get_conf:
with patch('aws.client.Client.get_s3_connection'):
with patch('aws.client.IDBroker.get_cab') as get_cab:
get_conf.return_value = {
'fs.s3a.ext.cab.address': 'address'
}
get_cab.return_value = {
'Credentials': {
'AccessKeyId': 'AccessKeyId',
'Expiration': 0
}
}
provider = get_credential_provider('default', 'hue')
assert_equal(
provider.get_credentials().get('AccessKeyId'),
'AccessKeyId')
client = Client.from_config(
conf.AWS_ACCOUNTS['default'],
get_credential_provider('default', 'hue'))
assert_equal(client._region, 'ap-northeast-1')
finally:
finish()
clear_cache()
conf.clear_cache()
def test_with_idbroker(self):
try:
finish = conf.AWS_ACCOUNTS.set_for_testing(
{}) # Set empty to test when no configs are set
with patch('aws.client.conf_idbroker.get_conf') as get_conf:
with patch('aws.client.Client.get_s3_connection'):
with patch('aws.client.IDBroker.get_cab') as get_cab:
get_conf.return_value = {
'fs.s3a.ext.cab.address': 'address'
}
get_cab.return_value = {
'Credentials': {
'AccessKeyId': 'AccessKeyId',
'Expiration': 0
}
}
provider = get_credential_provider('default', 'hue')
assert_equal(
provider.get_credentials().get('AccessKeyId'),
'AccessKeyId')
client1 = get_client(name='default',
fs='s3a',
user='******')
client2 = get_client(name='default',
fs='s3a',
user='******')
assert_not_equal(
client1, client2
) # Test that with Expiration 0 clients not equal
get_cab.return_value = {
'Credentials': {
'AccessKeyId':
'AccessKeyId',
'Expiration':
int(current_ms_from_utc()) + 10 * 1000
}
}
client3 = get_client(name='default',
fs='s3a',
user='******')
client4 = get_client(name='default',
fs='s3a',
user='******')
client5 = get_client(name='default',
fs='s3a',
user='******')
assert_equal(
client3, client4
) # Test that with 10 sec expiration, clients equal
assert_not_equal(
client4, client5
) # Test different user have different clients
finally:
finish()
clear_cache()
conf.clear_cache()
def test_with_core_site(self):
try:
finish = (conf.AZURE_ACCOUNTS.set_for_testing({'default': {}}),
conf.ABFS_CLUSTERS.set_for_testing({
'default': {
'fs_defaultfs': 'fs_defaultfs',
'webhdfs_url': 'webhdfs_url'
}
}))
with patch('azure.client.conf_idbroker.get_conf') as get_conf:
with patch('azure.client.ABFS.get_client'):
with patch('azure.client.ActiveDirectory.get_token'
) as get_token:
with patch('azure.conf.core_site.get_conf'
) as core_site_get_conf:
get_token.return_value = {
'access_token': 'access_token',
'token_type': '',
'expires_on': None
}
get_conf.return_value = {}
core_site_get_conf.return_value = {
'fs.azure.account.oauth2.client.id':
'client_id',
'fs.azure.account.oauth2.client.secret':
'client_secret',
'fs.azure.account.oauth2.client.endpoint':
'refresh_url'
}
client1 = get_client(name='default', fs='abfs')
client2 = get_client(name='default',
fs='abfs',
user='******')
provider = get_credential_provider(
'default', 'hue')
assert_equal(
provider.get_credentials().get('access_token'),
'access_token')
assert_equal(
client1, client2
) # Should be the same as no support for user based client with credentials & no Expiration
finally:
for f in finish:
f()
clear_cache()
def test_with_raz_enabled(self):
with patch('aws.client.RazS3Connection') as raz_s3_connection:
resets = [
RAZ.IS_ENABLED.set_for_testing(True),
conf.AWS_ACCOUNTS.set_for_testing({
'default': {
'region': 'us-west-2',
'host': 's3-us-west-2.amazonaws.com',
'allow_environment_credentials': 'false'
}
})
]
try:
client = get_client(name='default', fs='s3a', user='******')
assert_true(client)
finally:
for reset in resets:
reset()
clear_cache()
conf.clear_cache()
def clear_sys_caches(): return cluster.clear_caches(), fsmanager.clear_cache()
def clear_sys_caches():
return cluster.clear_caches(), fsmanager.clear_cache()
def test_with_idbroker(self):
try:
finish = (conf.AZURE_ACCOUNTS.set_for_testing({}),
conf.ADLS_CLUSTERS.set_for_testing({
'default': {
'fs_defaultfs': 'fs_defaultfs',
'webhdfs_url': 'webhdfs_url'
}
}))
with patch('azure.client.conf_idbroker.get_conf') as get_conf:
with patch('azure.client.WebHdfs.get_client'):
with patch('azure.client.IDBroker.get_cab') as get_cab:
get_conf.return_value = {
'fs.azure.ext.cab.address': 'address'
}
get_cab.return_value = {
'access_token': 'access_token',
'token_type': 'token_type',
'expires_on': 0
}
provider = get_credential_provider('default', 'hue')
assert_equal(
provider.get_credentials().get('access_token'),
'access_token')
client1 = get_client(name='default',
fs='adl',
user='******')
client2 = get_client(name='default',
fs='adl',
user='******')
assert_not_equal(
client1, client2
) # Test that with Expiration 0 clients not equal
get_cab.return_value = {
'Credentials': {
'access_token':
'access_token',
'token_type':
'token_type',
'expires_on':
int(current_ms_from_utc()) + 10 * 1000
}
}
client3 = get_client(name='default',
fs='adl',
user='******')
client4 = get_client(name='default',
fs='adl',
user='******')
client5 = get_client(name='default',
fs='adl',
user='******')
assert_equal(
client3, client4
) # Test that with 10 sec expiration, clients equal
assert_not_equal(
client4, client5
) # Test different user have different clients
finally:
for f in finish:
f()
clear_cache()
