def test_with_credentials(self): try: finish = conf.AWS_ACCOUNTS.set_for_testing({ 'default': { 'access_key_id': 'access_key_id', 'secret_access_key': 'secret_access_key' } }) with patch('aws.client.conf_idbroker.get_conf') as get_conf: with patch('aws.client.Client.get_s3_connection'): get_conf.return_value = {} client1 = get_client(name='default', fs='s3a') client2 = get_client(name='default', fs='s3a', user='******') provider = get_credential_provider('default', 'hue') assert_equal( provider.get_credentials().get('AccessKeyId'), conf.AWS_ACCOUNTS['default'].ACCESS_KEY_ID.get()) assert_equal( client1, client2 ) # Should be the same as no support for user based client with credentials & no Expiration finally: finish() clear_cache() conf.clear_cache()
def test_with_idbroker_on_ec2(self): try: finish = conf.AWS_ACCOUNTS.set_for_testing( {}) # Set empty to test when no configs are set with patch('aws.client.aws_conf.get_region') as get_region: with patch('aws.client.conf_idbroker.get_conf') as get_conf: with patch('aws.client.Client.get_s3_connection'): with patch('aws.client.IDBroker.get_cab') as get_cab: get_region.return_value = 'us-west-1' get_conf.return_value = { 'fs.s3a.ext.cab.address': 'address' } get_cab.return_value = { 'Credentials': { 'AccessKeyId': 'AccessKeyId', 'Expiration': 0 } } client = Client.from_config( None, get_credential_provider('default', 'hue')) assert_equal( client._region, 'us-west-1' ) # Test different user have different clients finally: finish() clear_cache() conf.clear_cache()
def test_with_idbroker_and_config(self): try: finish = conf.AWS_ACCOUNTS.set_for_testing( {'default': { 'region': 'ap-northeast-1' }}) with patch('aws.client.conf_idbroker.get_conf') as get_conf: with patch('aws.client.Client.get_s3_connection'): with patch('aws.client.IDBroker.get_cab') as get_cab: get_conf.return_value = { 'fs.s3a.ext.cab.address': 'address' } get_cab.return_value = { 'Credentials': { 'AccessKeyId': 'AccessKeyId', 'Expiration': 0 } } provider = get_credential_provider('default', 'hue') assert_equal( provider.get_credentials().get('AccessKeyId'), 'AccessKeyId') client = Client.from_config( conf.AWS_ACCOUNTS['default'], get_credential_provider('default', 'hue')) assert_equal(client._region, 'ap-northeast-1') finally: finish() clear_cache() conf.clear_cache()
def test_with_idbroker(self): try: finish = conf.AWS_ACCOUNTS.set_for_testing( {}) # Set empty to test when no configs are set with patch('aws.client.conf_idbroker.get_conf') as get_conf: with patch('aws.client.Client.get_s3_connection'): with patch('aws.client.IDBroker.get_cab') as get_cab: get_conf.return_value = { 'fs.s3a.ext.cab.address': 'address' } get_cab.return_value = { 'Credentials': { 'AccessKeyId': 'AccessKeyId', 'Expiration': 0 } } provider = get_credential_provider('default', 'hue') assert_equal( provider.get_credentials().get('AccessKeyId'), 'AccessKeyId') client1 = get_client(name='default', fs='s3a', user='******') client2 = get_client(name='default', fs='s3a', user='******') assert_not_equal( client1, client2 ) # Test that with Expiration 0 clients not equal get_cab.return_value = { 'Credentials': { 'AccessKeyId': 'AccessKeyId', 'Expiration': int(current_ms_from_utc()) + 10 * 1000 } } client3 = get_client(name='default', fs='s3a', user='******') client4 = get_client(name='default', fs='s3a', user='******') client5 = get_client(name='default', fs='s3a', user='******') assert_equal( client3, client4 ) # Test that with 10 sec expiration, clients equal assert_not_equal( client4, client5 ) # Test different user have different clients finally: finish() clear_cache() conf.clear_cache()
def test_with_core_site(self): try: finish = (conf.AZURE_ACCOUNTS.set_for_testing({'default': {}}), conf.ABFS_CLUSTERS.set_for_testing({ 'default': { 'fs_defaultfs': 'fs_defaultfs', 'webhdfs_url': 'webhdfs_url' } })) with patch('azure.client.conf_idbroker.get_conf') as get_conf: with patch('azure.client.ABFS.get_client'): with patch('azure.client.ActiveDirectory.get_token' ) as get_token: with patch('azure.conf.core_site.get_conf' ) as core_site_get_conf: get_token.return_value = { 'access_token': 'access_token', 'token_type': '', 'expires_on': None } get_conf.return_value = {} core_site_get_conf.return_value = { 'fs.azure.account.oauth2.client.id': 'client_id', 'fs.azure.account.oauth2.client.secret': 'client_secret', 'fs.azure.account.oauth2.client.endpoint': 'refresh_url' } client1 = get_client(name='default', fs='abfs') client2 = get_client(name='default', fs='abfs', user='******') provider = get_credential_provider( 'default', 'hue') assert_equal( provider.get_credentials().get('access_token'), 'access_token') assert_equal( client1, client2 ) # Should be the same as no support for user based client with credentials & no Expiration finally: for f in finish: f() clear_cache()
def test_with_raz_enabled(self): with patch('aws.client.RazS3Connection') as raz_s3_connection: resets = [ RAZ.IS_ENABLED.set_for_testing(True), conf.AWS_ACCOUNTS.set_for_testing({ 'default': { 'region': 'us-west-2', 'host': 's3-us-west-2.amazonaws.com', 'allow_environment_credentials': 'false' } }) ] try: client = get_client(name='default', fs='s3a', user='******') assert_true(client) finally: for reset in resets: reset() clear_cache() conf.clear_cache()
def clear_sys_caches(): return cluster.clear_caches(), fsmanager.clear_cache()
def test_with_idbroker(self): try: finish = (conf.AZURE_ACCOUNTS.set_for_testing({}), conf.ADLS_CLUSTERS.set_for_testing({ 'default': { 'fs_defaultfs': 'fs_defaultfs', 'webhdfs_url': 'webhdfs_url' } })) with patch('azure.client.conf_idbroker.get_conf') as get_conf: with patch('azure.client.WebHdfs.get_client'): with patch('azure.client.IDBroker.get_cab') as get_cab: get_conf.return_value = { 'fs.azure.ext.cab.address': 'address' } get_cab.return_value = { 'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': 0 } provider = get_credential_provider('default', 'hue') assert_equal( provider.get_credentials().get('access_token'), 'access_token') client1 = get_client(name='default', fs='adl', user='******') client2 = get_client(name='default', fs='adl', user='******') assert_not_equal( client1, client2 ) # Test that with Expiration 0 clients not equal get_cab.return_value = { 'Credentials': { 'access_token': 'access_token', 'token_type': 'token_type', 'expires_on': int(current_ms_from_utc()) + 10 * 1000 } } client3 = get_client(name='default', fs='adl', user='******') client4 = get_client(name='default', fs='adl', user='******') client5 = get_client(name='default', fs='adl', user='******') assert_equal( client3, client4 ) # Test that with 10 sec expiration, clients equal assert_not_equal( client4, client5 ) # Test different user have different clients finally: for f in finish: f() clear_cache()