def refreshable_session(self) -> Tuple[Session, bool]:
"""
Get refreshable boto3 session.
"""
try:
LOG.debug("Creating refreshable credentials!")
# get refreshable credentials
refreshable_credentials = RefreshableCredentials.create_from_metadata(
metadata=self.__get_session_credentials(),
refresh_using=self.__get_session_credentials,
method="sts-assume-role",
)
# attach refreshable credentials current session
session = get_session()
session._credentials = refreshable_credentials
session.set_config_variable("region", self.region_name)
autorefresh_session = Session(botocore_session=session)
return autorefresh_session, True
except Exception as ex:
LOG.error(
f"Got an exception when creating refreshable credentials! ex={ex}"
)
return Session(), False
def refreshed_session(region='us-east-1'):
"""Assume a boto3.Session With automatic credential renewal.
NOTE: We have to poke at botocore internals a few times.
Args:
region (str, optional): The region of the session.
Defaults to 'us-east-1'.
Returns:
session (Session): an boto3 session with RefreshableCredentials
"""
def _refresh():
credentials = Session().get_credentials()
# set the expiry time to one hour from now.
# Note this is not the real session expiry time,
# but we are focusing refresh every 1 hour.
return dict(
access_key=credentials.access_key,
secret_key=credentials.secret_key,
token=credentials.token,
expiry_time=(pytz.utc.localize(datetime.utcnow()) + timedelta(hours=1)).isoformat())
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=_refresh(),
refresh_using=_refresh,
method='session-cred')
re_session = get_session()
re_session._credentials = session_credentials
re_session.set_config_variable('region', region)
return Session(botocore_session=re_session)
def _get_credentials(cls, account_id: str, region: str) -> RefreshableCredentials:
"""
Retrieves refreshable credentials for the given account.
The credentials are cached using a LRU cache.
Args:
account_id: The id of the account
"""
cls.logger.info("Getting credentials for accountId %s and region %s", account_id, region)
def refresh_credentials() -> Dict[str, str]:
"""Refresh credentials by invoking STS AssumeRole operation"""
cls.logger.info("Refreshing credentials for account %s and region %s", account_id, region)
params = {
'RoleArn': 'arn:aws:iam::{}:role/PantherRemediationRole-{}'.format(account_id, _MASTER_REGION),
'RoleSessionName': 'RemediationSession',
'DurationSeconds': 3600,
}
response = cls._get_sts_client(region).assume_role(**params).get('Credentials')
return {
'access_key': response.get('AccessKeyId'),
'secret_key': response.get('SecretAccessKey'),
'token': response.get('SessionToken'),
'expiry_time': response.get('Expiration').isoformat(),
}
return RefreshableCredentials.create_from_metadata(
metadata=refresh_credentials(),
refresh_using=refresh_credentials,
method='sts-assume-role',
)
def __get_session_credentials(self):
"""
Get session credentials
"""
fetcher = self._role_fetcher
# We do the first request, to see if we get useful data back.
# If not, we'll pass & move on to whatever's next in the credential
# chain.
metadata = fetcher.retrieve_iam_role_credentials()
if not metadata:
return None
logging.debug('Found credentials from IAM Role: %s',
metadata['role_name'])
# We manually set the data here, since we already made the request &
# have it. When the expiry is hit, the credentials will auto-refresh
# themselves.
credentials = RefreshableCredentials.create_from_metadata(
metadata,
method=self.METHOD,
refresh_using=fetcher.retrieve_iam_role_credentials,
)
self.access_key = credentials.access_key
self.secret_key = credentials.secret_key
self.security_token = credentials.token
return credentials
def as_boto3_s3_client(self):
"""
Convert the Optimizely S3 client to a standard boto s3
client with refreshable credentials
Returns:
botocore.client.S3: Standard boto3 S3 client
"""
creds = self.get_creds()
# The API response is in milliseconds
expiry_time = int(creds['expiration'] / 1000)
# boto expects the expiry time to be a UTC datetime
expiry_time = datetime.fromtimestamp(expiry_time, pytz.utc)
opz_refreshable_credentials = RefreshableCredentials(
creds['accessKeyId'], creds['secretAccessKey'],
creds['sessionToken'], expiry_time, self.get_creds,
OptimizelyS3Client.CREDENTIALS_IDENTIFIER)
session = get_session()
session._credentials = opz_refreshable_credentials
session.set_config_variable(
'region', OptimizelyS3Client.EVENT_EXPORT_BUCKET_REGION)
opz_session = boto3.Session(botocore_session=session)
s3_client = opz_session.client('s3')
return s3_client
def assumed_session(oidc_profile,
endpoint="https://minio.cloud.infn.it/",
verify=True,
session=None):
"""STS Role assume a boto3.Session
With automatic credential renewal.
Args:
oidc_profile: the name of the oidc-agent profile to be used for the identity provider
session: an optional extant session, note session is captured
in a function closure for renewing the sts assumed role.
Notes: We have to poke at botocore internals a few times
"""
if session is None:
session = Session()
def refresh():
creds = s3_session_credentials(oidc_profile,
endpoint=endpoint,
verify=verify)
return creds
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=refresh(), refresh_using=refresh, method='sts')
# so dirty.. it hurts, no clean way to set this outside of the internals poke
s = get_session()
s._credentials = session_credentials
return Session(botocore_session=s)
def refresh():
global credential_loader
credential_loader = RefreshableCredentials.create_from_metadata(
metadata=gimme_aws_creds_as_metadata(),
method='gimme-aws-creds',
refresh_using=gimme_aws_creds_as_metadata,
)
return '', http.HTTPStatus.NO_CONTENT
def load(self):
creds = RefreshableCredentials.create_from_metadata(
metadata=self._credentials,
method=self.METHOD,
refresh_using=_refresh_credentials,
)
return creds
def assumed_session(role_arn,
session_name,
session=None,
region=None,
external_id=None):
"""STS Role assume a boto3.Session
With automatic credential renewal.
Args:
role_arn: iam role arn to assume
session_name: client session identifier
session: an optional extant session, note session is captured
in a function closure for renewing the sts assumed role.
:return: a boto3 session using the sts assumed role credentials
Notes: We have to poke at botocore internals a few times
"""
if session is None:
session = Session()
retry = get_retry(('Throttling', ))
def refresh():
parameters = {"RoleArn": role_arn, "RoleSessionName": session_name}
if external_id is not None:
parameters['ExternalId'] = external_id
credentials = retry(
session.client(
'sts', endpoint_url=sts_regional_endpoint(region)).assume_role,
**parameters)['Credentials']
return dict(
access_key=credentials['AccessKeyId'],
secret_key=credentials['SecretAccessKey'],
token=credentials['SessionToken'],
# Silly that we basically stringify so it can be parsed again
expiry_time=credentials['Expiration'].isoformat())
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=refresh(), refresh_using=refresh, method='sts-assume-role')
# so dirty.. it hurts, no clean way to set this outside of the
# internals poke. There's some work upstream on making this nicer
# but its pretty baroque as well with upstream support.
# https://github.com/boto/boto3/issues/443
# https://github.com/boto/botocore/issues/761
s = get_session()
s._credentials = session_credentials
if region is None:
region = s.get_config_variable('region') or 'us-east-1'
s.set_config_variable('region', region)
return Session(botocore_session=s)
def __create_session(self):
session_creds = RefreshableCredentials.create_from_metadata(
metadata=self.__sts_role_credentials(),
refresh_using=self.__sts_role_credentials,
method="sts-assume-role",
)
session = get_session()
session._credentials = session_creds
session.set_config_variable("region", self.region)
return boto3.Session(botocore_session=session)
def assumed_session(role_arn, session_name, session=None, region=None, external_id=None):
"""STS Role assume a boto3.Session
With automatic credential renewal.
Args:
role_arn: iam role arn to assume
session_name: client session identifier
session: an optional extant session, note session is captured
in a function closure for renewing the sts assumed role.
:return: a boto3 session using the sts assumed role credentials
Notes: We have to poke at botocore internals a few times
"""
if session is None:
session = Session()
retry = get_retry(('Throttling',))
def refresh():
parameters = {"RoleArn": role_arn, "RoleSessionName": session_name}
if external_id is not None:
parameters['ExternalId'] = external_id
credentials = retry(
session.client('sts').assume_role, **parameters)['Credentials']
return dict(
access_key=credentials['AccessKeyId'],
secret_key=credentials['SecretAccessKey'],
token=credentials['SessionToken'],
# Silly that we basically stringify so it can be parsed again
expiry_time=credentials['Expiration'].isoformat())
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=refresh(),
refresh_using=refresh,
method='sts-assume-role')
# so dirty.. it hurts, no clean way to set this outside of the
# internals poke. There's some work upstream on making this nicer
# but its pretty baroque as well with upstream support.
# https://github.com/boto/boto3/issues/443
# https://github.com/boto/botocore/issues/761
s = get_session()
s._credentials = session_credentials
if region is None:
region = s.get_config_variable('region') or 'us-east-1'
s.set_config_variable('region', region)
return Session(botocore_session=s)
def create_session(self):
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=self.refresh_creds(),
refresh_using=self.refresh_creds,
method="sts-assume-role",
)
session = get_session()
session._credentials = session_credentials
session.set_config_variable("region", self.region)
autorefresh_session = boto3.Session(botocore_session=session)
self.auto_refresh_session_step_func = autorefresh_session.client(
"stepfunctions", region_name=self.region)
def _get_refreshable_session(self):
"""
This function returns a refreshable session for a given role and a session name
"""
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=self._refresh_credentials(),
refresh_using=self._refresh_credentials,
method='sts-assume-role')
session = get_session()
session._credentials = session_credentials
auto_refresh_session = boto3.Session(botocore_session=session)
return auto_refresh_session
def load(self):
"""
Search for credentials with client_grants
"""
if self.cid is not None:
fetcher = self._create_credentials_fetcher()
return RefreshableCredentials.create_from_metadata(
metadata=fetcher(),
refresh_using=fetcher,
method=self.METHOD,
)
else:
return None
def load(self):
if self.config:
fetcher = self._create_credentials_fetcher()
credentials = fetcher(time_as="datetime")
res = RefreshableCredentials(credentials["access_key"],
credentials["secret_key"],
credentials["token"],
credentials["expiry_time"],
refresh_using=fetcher,
method=self.METHOD)
else:
res = None
return res
def _call_paginator(self, metric):
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=self._refresh(),
refresh_using=self._refresh,
method="sts-assume-role",
)
session = get_session()
session._credentials = session_credentials
autorefresh_session = Session(botocore_session=session)
service = autorefresh_session.client(metric.service)
# service = self._session.client(metric.service)
paginator = service.get_paginator(metric.method)
paginate_response_iterator = paginator.paginate(**metric.method_args)
return list(paginate_response_iterator.search(metric.search))
def assume_role(self):
''' Assume the role. Uses SAML if requested. '''
if self.__authType == 'InstanceProfile':
return botoSession(region_name=self.__region)
elif self.__useSAML == True:
self.__setIDPAuthUrlAndPayload__()
self.__setSAMLAssertion__()
self.__decideThePrincipalArn__()
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=self.__refreshCred__(),
refresh_using=self.__refreshCred__,
method='sts-assume-role-with-saml')
else:
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=self.__refreshCred__(),
refresh_using=self.__refreshCred__,
method='sts-assume-role')
s = get_session()
s._credentials = session_credentials
if self.__region is None:
self.__region = s.get_config_variable('region')
s.set_config_variable('region', self.__region)
return botoSession(botocore_session=s)
def __init__(self, session_name, role_arn, region='us-east-1'):
self.role_arn = role_arn
self.session_name = session_name
self.region = region
self.sts_client = boto3.client('sts')
credentials = self._refresh()
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=credentials,
refresh_using=self._refresh,
method="sts-assume-role")
session = get_session()
session._credentials = session_credentials
session.set_config_variable("region", region)
self.session = boto3.Session(botocore_session=session)
def load(self):
fetcher = self._fetch_metadata
metadata = fetcher()
if not metadata:
return None
log.debug("Obtained for account %s will expire at %s",
self.metadata['account_id'], self.credentials['expiration'])
return RefreshableCredentials.create_from_metadata(
metadata,
method=self.METHOD,
refresh_using=fetcher,
)
def _get_credentials(self):
if self.credentials is None:
sts = boto3.client('sts', region_name=self.region)
role = sts.assume_role(RoleArn=self.role_arn,
RoleSessionName=uuid.uuid4().hex,
DurationSeconds=3600)
self.credentials = RefreshableCredentials.create_from_metadata(
metadata=self._new_credentials(),
refresh_using=self._new_credentials,
method="sts-assume-role",
)
return self.credentials.get_frozen_credentials()
def oktadboto_session(profile,
expiration=timedelta(minutes=50),
region="us-east-1"):
@wraps(get_oktad_credentials)
def wrap_refresh():
return get_oktad_credentials(profile, expiration)
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=wrap_refresh(),
refresh_using=wrap_refresh,
method="sts-assume-role",
)
session = get_session()
session._credentials = session_credentials
session.set_config_variable("region", region)
return Session(botocore_session=session)
def __init__(self, target_role_arn: str, region: str):
sts_client = boto3.client(
'sts',
region_name=region,
endpoint_url="https://sts.{}.amazonaws.com".format(region))
self.sts_client = sts_client
session_name = sts_client.get_caller_identity()['Arn'].split(
":")[5].split("/")[1]
self.role_arn = target_role_arn
self.session_name = session_name
# Create Refrehable Credentials
refreshable = RefreshableCredentials.create_from_metadata(
metadata=self._refresh(),
refresh_using=self._refresh,
method="sts-assume-role")
self.refreshable_cred = refreshable
pass
def assumed_session(role_arn, session_name, session=None, region=None):
"""STS Role assume a boto3.Session
With automatic credential renewal.
Args:
role_arn: iam role arn to assume
session_name: client session identifier
session: an optional extant session, note session is captured
in a function closure for renewing the sts assumed role.
:return: a boto3 session using the sts assumed role credentials
Notes: Borrowed from
https://github.com/cloud-custodian/cloud-custodian/blob/master/c7n/credentials.py
"""
if session is None:
session = Session()
def refresh():
credentials = session.client("sts").assume_role(
RoleArn=role_arn, RoleSessionName=session_name)["Credentials"]
return dict(
access_key=credentials["AccessKeyId"],
secret_key=credentials["SecretAccessKey"],
token=credentials["SessionToken"],
# Silly that we basically stringify so it can be parsed again
expiry_time=credentials["Expiration"].isoformat(),
)
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=refresh(), refresh_using=refresh, method="sts-assume-role")
# so dirty.. it hurts, no clean way to set this outside of the
# internals poke. There's some work upstream on making this nicer
# but its pretty baroque as well with upstream support.
# https://github.com/boto/boto3/issues/443
# https://github.com/boto/botocore/issues/761
s = get_session()
s._credentials = session_credentials
if region is None:
region = s.get_config_variable("region") or "us-east-1"
s.set_config_variable("region", region)
return Session(botocore_session=s)
def get_s3_client_refreshable(refresh_method):
"""Return thread-safe s3 client with refreshable credentials.
:param refresh_method: function that can get fresh credentials
"""
session = get_session()
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=refresh_method(),
refresh_using=refresh_method,
method="sts-assume-role",
)
# pylint: disable=protected-access
session._credentials = session_credentials
boto3_session = boto3.Session(botocore_session=session)
return boto3_session.client(
"s3",
endpoint_url=os.environ.get("LOCALSTACK_S3_ENDPOINT") or None,
)
def get_autorefresh_session(**args):
""" return auto referesh session using assume role
Usage:
params = {
"DurationSeconds": os.getenv('SESSION_DURATION', 3600),
"RoleArn": os.getenv('AWS_ROLE_ARN'),
"RoleSessionName": os.getenv('AWS_SESSION_NAME', 'test_session'),
"WebIdentityToken": open(os.getenv('AWS_WEB_IDENTITY_TOKEN_FILE')).read(),
}
session = autorefresh_session(**params)
client = session.client('sqs')
"""
session = get_session()
session._credentials = RefreshableCredentials.create_from_metadata(
metadata=refresh_credentials(**args),
refresh_using=refresh_credentials,
method="sts-assume-role-with-web-identity",
)
return boto3.Session(botocore_session=session)
def __init__(self,
arn: str,
session_name: str = unique(),
aws_region: str = AWS_REGION,
duration=3600):
self._arn = arn
self._session_name = session_name
self._aws_region = aws_region
self._duration = duration
self._sts_client = self.base_session.client(
"sts", region_name=self._aws_region)
session = get_session()
session._credentials = RefreshableCredentials.create_from_metadata(
metadata=self._refresh(),
refresh_using=self._refresh,
method="sts-assume-role",
)
session.set_config_variable("region", self._aws_region)
self.session = Session(botocore_session=session)
def assumed_session(cls, source_bucket_id, destination_bucket_id, command, versioning=False):
def refresh():
credentials = DataStorage.get_temporary_credentials(source_bucket_id, destination_bucket_id, command,
versioning=versioning)
return dict(
access_key=credentials.access_key_id,
secret_key=credentials.secret_key,
token=credentials.session_token,
expiry_time=credentials.expiration,
region_name=credentials.region)
fresh_metadata = refresh()
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=fresh_metadata,
refresh_using=refresh,
method='sts-assume-role')
s = get_session()
s._credentials = session_credentials
return Session(botocore_session=s, region_name=fresh_metadata['region_name'])
def _call_service_method(self, metric):
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=self._refresh(),
refresh_using=self._refresh,
method="sts-assume-role",
)
session = get_session()
session._credentials = session_credentials
autorefresh_session = Session(botocore_session=session)
service = autorefresh_session.client(metric.service)
# service = self._session.client(metric.service)
service_method = getattr(service, metric.method)
next_token = ''
responses = []
kwargs = dict(**metric.method_args)
while next_token is not None:
response = service_method(**kwargs)
next_token = response.get('NextToken', None)
responses += jmespath.search(metric.search, response)
kwargs["NextToken"] = next_token
return responses
def refreshable_session(self) -> Session:
"""
Get refreshable boto3 session.
"""
try:
# get refreshable credentials
refreshable_credentials = RefreshableCredentials.create_from_metadata(
metadata=self.__get_session_credentials(),
refresh_using=self._role_fetcher.retrieve_iam_role_credentials,
method=self.METHOD,
)
# attach refreshable credentials current session
session = get_session()
session._credentials = refreshable_credentials
session.set_config_variable("region", self.region_name)
autorefresh_session = Session(botocore_session=session)
return autorefresh_session
except:
return boto3.session.Session()
def assumed_session(role_arn, session_name, session=None):
"""STS Role assume a boto3.Session
With automatic credential renewal.
Args:
role_arn: iam role arn to assume
session_name: client session identifier
session: an optional extant session, note session is captured
in a function closure for renewing the sts assumed role.
Notes: We have to poke at botocore internals a few times
"""
if session is None:
session = Session()
def refresh():
credentials = session.client('sts').assume_role(
RoleArn=role_arn,
RoleSessionName=session_name)['Credentials']
return dict(
access_key=credentials['AccessKeyId'],
secret_key=credentials['SecretAccessKey'],
token=credentials['SessionToken'],
# Silly that we basically stringify so it can be parsed again
expiry_time=credentials['Expiration'].isoformat())
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=refresh(),
refresh_using=refresh,
method='sts-assume-role')
# so dirty.. it hurts, no clean way to set this outside of the internals poke
s = get_session()
s._credentials = session_credentials
region = session._session.get_config_variable('region') or 'us-east-1'
s.set_config_variable('region', region)
return Session(botocore_session=s)
def get_s3(obj="resource"):
global AUTOREFRESH_SESSION
global ROLE_ARN
if ROLE_ARN not in S3_CLIENT:
S3_CLIENT[ROLE_ARN] = dict()
if ROLE_ARN == "default":
if AUTOREFRESH_SESSION is None:
AUTOREFRESH_SESSION = boto3.Session()
S3_CLIENT["default"]["resource"] = AUTOREFRESH_SESSION.resource('s3')
S3_CLIENT["default"]["client"] = AUTOREFRESH_SESSION.client('s3')
else:
if AUTOREFRESH_SESSION is None:
sess = get_session()
sess._credentials = RefreshableCredentials.create_from_metadata(
metadata=refresh_sts_session_keys(),
refresh_using=refresh_sts_session_keys,
method="sts-assume-role")
AUTOREFRESH_SESSION = boto3.Session(botocore_session=sess)
S3_CLIENT[ROLE_ARN]["resource"] = AUTOREFRESH_SESSION.resource("s3")
S3_CLIENT[ROLE_ARN]["client"] = AUTOREFRESH_SESSION.client("s3")
return S3_CLIENT.get(ROLE_ARN, dict()).get(obj)