def __init__(self, credentials_provider, transfer_acceleration=True):
config = Config(s3={'use_accelerate_endpoint': True}) if transfer_acceleration else Config()
botocore_session = get_session()
botocore_session.register_component('credential_provider', CredentialResolver(providers=[credentials_provider]))
my_session = boto3.Session(botocore_session=botocore_session)
self.target_s3 = my_session.resource('s3', config=config)
self.source_s3_client = boto3.client('s3')
def create_botocore_session():
botocore_session = botocore.session.get_session()
# If we have saved credentials, use them. Otherwise, create a normal Boto session.
credentials = _load_credentials()
if credentials:
provider = QuiltProvider(credentials)
resolver = CredentialResolver([provider])
botocore_session.register_component('credential_provider', resolver)
return botocore_session
def make_refreshable_assume_role_session(main_session, assume_role_params):
provider = SessionWithRefreshableAssumeRoleProvider(
main_session, assume_role_params)
resolver = CredentialResolver(providers=[provider])
botocore_session = get_session()
botocore_session.register_component('credential_provider', resolver)
return Session(botocore_session=botocore_session,
region_name=main_session.region_name)
def assume_role(session: Session,
role_arn: str,
duration: int = 3600,
session_name: str = None) -> Session:
# noinspection PyTypeChecker
fetcher = AssumeRoleCredentialFetcher(session.create_client,
session.get_credentials(),
role_arn,
extra_args={
'DurationSeconds': duration,
'RoleSessionName': session_name
})
role_session = Session()
role_session.register_component(
'credential_provider',
CredentialResolver([AssumeRoleProvider(fetcher)]))
return role_session
def setup_aws_client(config):
role_arn = "arn:aws:iam::{}:role/{}".format(
config['account_id'].replace('-', ''), config['role_name'])
session = Session()
fetcher = AssumeRoleCredentialFetcher(session.create_client,
session.get_credentials(),
role_arn,
extra_args={
'DurationSeconds': 3600,
'RoleSessionName': 'TapS3CSV',
'ExternalId':
config['external_id']
},
cache=JSONFileCache())
refreshable_session = Session()
refreshable_session.register_component(
'credential_provider',
CredentialResolver([AssumeRoleProvider(fetcher)]))
LOGGER.info("Attempting to assume_role on RoleArn: %s", role_arn)
boto3.setup_default_session(botocore_session=refreshable_session)
def _get_boto3_session(region: str,
role_arn: str = None,
assume_duration: int = 3600) -> Session:
"""Creates a boto3 session, optionally assuming a role.
Args:
region: The AWS region for the session.
role_arn: The ARN to assume for the session.
assume_duration: The duration (in seconds) to assume the role.
Returns:
object: A boto3 Session.
"""
# By default return a basic session
if not role_arn:
return Session(region_name=region)
# The following assume role example was taken from
# https://github.com/boto/botocore/issues/761#issuecomment-426037853
# Create a session used to assume role
assume_session = BotocoreSession()
fetcher = AssumeRoleCredentialFetcher(
assume_session.create_client,
assume_session.get_credentials(),
role_arn,
extra_args={
"DurationSeconds": assume_duration,
},
cache=JSONFileCache(),
)
role_session = BotocoreSession()
role_session.register_component(
"credential_provider",
CredentialResolver([Boto3Manager.AssumeRoleProvider(fetcher)]),
)
return Session(region_name=region, botocore_session=role_session)
def create_credential_resolver(session):
"""Create a default credential resolver.
This creates a pre-configured credential resolver
that includes the default lookup chain for
credentials.
"""
profile_name = session.get_config_variable('profile')
credential_file = session.get_config_variable('credentials_file')
config_file = session.get_config_variable('config_file')
metadata_timeout = session.get_config_variable(
'metadata_service_timeout')
num_attempts = session.get_config_variable(
'metadata_service_num_attempts')
providers = []
if profile_name is None:
providers += [
EnvProvider(),
]
profile_name = 'default'
providers += [
# The new config file has precedence over the legacy
# config file.
SharedCredentialProvider(creds_filename=credential_file,
profile_name=profile_name),
# The new config file has precedence over the legacy
# config file.
ConfigProvider(config_filename=config_file,
profile_name=profile_name),
OriginalEC2Provider(),
BotoProvider(),
InstanceMetadataProvider(iam_role_fetcher=InstanceMetadataFetcher(
timeout=metadata_timeout, num_attempts=num_attempts))
]
resolver = CredentialResolver(providers=providers)
return resolver
def setup_aws_client(config):
role_arn = "arn:aws:iam::{}:role/{}".format(
config["account_id"].replace("-", ""), config["role_name"])
session = Session()
fetcher = AssumeRoleCredentialFetcher(
session.create_client,
session.get_credentials(),
role_arn,
extra_args={
"DurationSeconds": 3600,
"RoleSessionName": "TapS3CSV",
"ExternalId": config["external_id"],
},
cache=JSONFileCache(),
)
refreshable_session = Session()
refreshable_session.register_component(
"credential_provider",
CredentialResolver([AssumeRoleProvider(fetcher)]))
LOGGER.info("Attempting to assume_role on RoleArn: %s", role_arn)
boto3.setup_default_session(botocore_session=refreshable_session)
def setup_aws_client(config):
if 'role_name' in config:
role_arn = "arn:aws:iam::{}:role/{}".format(
config['account_id'].replace('-', ''), config['role_name'])
session = Session()
fetcher = AssumeRoleCredentialFetcher(session.create_client,
session.get_credentials(),
role_arn,
extra_args={
'DurationSeconds':
3600,
'RoleSessionName':
'TapDynamodDB',
'ExternalId':
config['external_id']
},
cache=JSONFileCache())
refreshable_session = Session()
refreshable_session.register_component(
'credential_provider',
CredentialResolver([AssumeRoleProvider(fetcher)]))
LOGGER.info("Attempting to assume_role on RoleArn: %s", role_arn)
boto3.setup_default_session(botocore_session=refreshable_session)
elif 'aws_access_key_id' in config and 'aws_secret_access_key' in config:
LOGGER.info(
"Attempting to pass AWS credentials from 'aws_access_key_id' and 'aws_secret_access_key' config values"
)
boto3.setup_default_session(
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
aws_session_token=config.get('aws_session_token', None))
session = Session()
def create_credential_resolver(session, cache=None, region_name=None):
"""Create a default credential resolver.
This creates a pre-configured credential resolver
that includes the default lookup chain for
credentials.
"""
profile_name = session.get_config_variable("profile") or "default"
metadata_timeout = session.get_config_variable("metadata_service_timeout")
num_attempts = session.get_config_variable("metadata_service_num_attempts")
disable_env_vars = session.instance_variables().get("profile") is not None
if cache is None:
cache = {}
env_provider = EnvProvider()
container_provider = ContainerProvider()
instance_metadata_provider = InstanceMetadataProvider(
iam_role_fetcher=InstanceMetadataFetcher(
timeout=metadata_timeout,
num_attempts=num_attempts,
user_agent=session.user_agent(),
))
profile_provider_builder = ProfileProviderBuilder(session,
cache=cache,
region_name=region_name)
assume_role_provider = AssumeRoleProvider(
load_config=lambda: session.full_config,
client_creator=_get_client_creator(session, region_name),
cache=cache,
profile_name=profile_name,
credential_sourcer=CanonicalNameCredentialSourcer(
[env_provider, container_provider, instance_metadata_provider]),
profile_provider_builder=profile_provider_builder,
)
pre_profile = [
env_provider,
assume_role_provider,
]
profile_providers = profile_provider_builder.providers(
profile_name=profile_name,
disable_env_vars=disable_env_vars,
)
post_profile = [
OriginalEC2Provider(),
BotoProvider(),
container_provider,
instance_metadata_provider,
]
providers = pre_profile + profile_providers + post_profile
if disable_env_vars:
# An explicitly provided profile will negate an EnvProvider.
# We will defer to providers that understand the "profile"
# concept to retrieve credentials.
# The one edge case if is all three values are provided via
# env vars:
# export AWS_ACCESS_KEY_ID=foo
# export AWS_SECRET_ACCESS_KEY=bar
# export AWS_PROFILE=baz
# Then, just like our client() calls, the explicit credentials
# will take precedence.
#
# This precedence is enforced by leaving the EnvProvider in the chain.
# This means that the only way a "profile" would win is if the
# EnvProvider does not return credentials, which is what we want
# in this scenario.
providers.remove(env_provider)
LOGGER.debug("Skipping environment variable credential check"
" because profile name was explicitly set.")
return CredentialResolver(providers=providers)