def get_credentials_for_assumed_role(self, access_key, secret_key, token):
results = OrderedDict()
self.logger.debug("Connecting to AWS region %s ...", self.region)
try:
conn = connect_to_region(self.region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
security_token=token,
proxy=self.aws_proxy_host,
proxy_port=self.aws_proxy_port
)
try:
response = conn.assume_role(
role_arn=self.role_to_assume,
role_session_name=self.get_session_name())
self.logger.info("Successfully got credentials for role: %s",
self.role_to_assume)
finally:
conn.close()
results[self.get_role_name()] = self.create_credentials_json(response)
except Exception:
self.logger.exception("Could not assume the AWS role '%s':",
self.role_to_assume)
return results
def sts_conn():
aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
return sts.connect_to_region(
'us-east-1',
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
def handle_security_credentials(role_name):
"""
Assumes an IAM role and returns the security credentials. Caches results in application context until expiration.
:param role_name: the IAM role to assume
"""
with app.app_context():
credentials = getattr(current_app, '_security_credentials', None)
expiration = getattr(current_app, '_security_credentials_expiration', None)
if not credentials or datetime.now(pytz.utc) > expiration:
role = iam.connect_to_region(REGION).get_role(role_name)
session = sts.connect_to_region(REGION).assume_role(role.arn, "Local")
credentials = jsonify({
"Code": "Success",
"LastUpdated": datetime.now(pytz.utc).strftime(DATE_FORMAT),
"Type": "AWS-HMAC",
"AccessKeyId": session.credentials.access_key,
"SecretAccessKey": session.credentials.secret_key,
"Token": session.credentials.session_token,
"Expiration": session.credentials.expiration
})
current_app._security_credentials = credentials
current_app._security_credentials_expiration = datetime.strptime(session.credentials.expiration,
DATE_FORMAT).replace(tzinfo=pytz.utc)
return credentials
def get_credentials_for(session_name, role_arn):
conn = sts.connect_to_region(
CONFIG.get('aws_region'),
aws_access_key_id=CONFIG.get('aws_access_key_id'),
aws_secret_access_key=CONFIG.get('aws_secret_access_key'))
if conn is None:
return 'Could not get connection', 500
c = conn.assume_role(role_arn=role_arn, role_session_name=session_name)
return c.credentials
def handle(self, *args, **options):
pprint(options)
if 'aws_access_key_id' in options and 'aws_secret_access_key' in options:
sts_conn = sts.connect_to_region(
'eu-west-1',
aws_access_key_id=options['aws_access_key_id'],
aws_secret_access_key=options['aws_secret_access_key'])
else:
sts_conn = sts.connect_to_region('eu-west-1')
credentials = sts_conn.assume_role(self.role_name, self.session_name)
pprint(credentials.credentials.__dict__)
conn = kinesis.connect_to_region(
'eu-west-1',
aws_access_key_id=credentials.credentials.access_key,
aws_secret_access_key=credentials.credentials.secret_key,
security_token=credentials.credentials.session_token)
streaminfo = conn.describe_stream(self.stream_name)
pprint(streaminfo)
threadlist = map(
lambda shardinfo: self.startup_thread(credentials.credentials,
shardinfo),
streaminfo['StreamDescription']['Shards'])
print "Stream {0} has {1} shards".format(self.stream_name,
len(threadlist))
for t in threadlist:
t.daemon = True
t.start()
print "Started up and processing. Hit CTRL-C to stop."
#simplest way to allow ctrl-C when dealing with threads
try:
while True:
sleep(3600)
except KeyboardInterrupt:
print "CTRL-C caught, cleaning up"
def get_credentials_for(session_name, role_arn):
conn = sts.connect_to_region(
CONFIG.get('aws_region'),
aws_access_key_id = CONFIG.get('aws_access_key_id'),
aws_secret_access_key = CONFIG.get('aws_secret_access_key'))
if conn is None:
return 'Could not get connection', 500
c = conn.assume_role(
role_arn = role_arn,
role_session_name = session_name)
return c.credentials
def refresh_access_credentials(self):
sts_conn = sts.connect_to_region(
'eu-west-1',
aws_access_key_id=self._aws_access_key_id,
aws_secret_access_key=self._aws_secret_access_key)
credentials = sts_conn.assume_role(self.role_name, self.session_name)
self._conn = kinesis.connect_to_region(
'eu-west-1',
aws_access_key_id=credentials.credentials.access_key,
aws_secret_access_key=credentials.credentials.secret_key,
security_token=credentials.credentials.session_token)
def getTemporaryCredentials(self,user):
"""
Gets token that allows for S3 Uploads for seconds set in STS_LIFETIME
"""
stsConnection = sts.connect_to_region(s3UrlHandler.REGION)
role = stsConnection.assume_role(s3UrlHandler.S3_ROLE,"FileUpload"+str(user),duration_seconds=s3UrlHandler.STS_LIFETIME)
credentials ={}
credentials["AccessKeyId"] = role.credentials.access_key
credentials["SecretAccessKey"] = role.credentials.secret_key
credentials["SessionToken"] = role.credentials.session_token
credentials["Expiration"] = role.credentials.expiration
return credentials
def get_temporary_credentials(user):
"""
Gets token that allows for S3 Uploads for seconds set in STS_LIFETIME
"""
sts_connection = sts.connect_to_region(S3Handler.REGION)
role = sts_connection.assume_role(S3Handler.S3_ROLE, "FileUpload" + str(user),
duration_seconds=S3Handler.STS_LIFETIME)
credentials = {
'AccessKeyId': role.credentials.access_key,
'SecretAccessKey': role.credentials.secret_key,
'SessionToken': role.credentials.session_token,
'Expiration': role.credentials.expiration
}
return credentials
def get_temporary_credentials(user):
"""
Gets token that allows for S3 Uploads for seconds set in STS_LIFETIME
"""
sts_connection = sts.connect_to_region(S3Handler.REGION)
role = sts_connection.assume_role(S3Handler.S3_ROLE, "FileUpload" + str(user),
duration_seconds=S3Handler.STS_LIFETIME)
credentials = {
'AccessKeyId': role.credentials.access_key,
'SecretAccessKey': role.credentials.secret_key,
'SessionToken': role.credentials.session_token,
'Expiration': role.credentials.expiration
}
return credentials
def get_s3_connection(self):
"""
Uses temporaray role credentials to connect to S3
:return:
"""
sts_conn = sts.connect_to_region(
'eu-west-1',
aws_access_key_id=getattr(settings, 'ATOM_RESPONDER_AWS_KEY_ID',
None),
aws_secret_access_key=getattr(settings, 'ATOM_RESPONDER_SECRET',
None))
credentials = sts_conn.assume_role(self.role_name, self.session_name)
return s3.connect_to_region(
'eu-west-1',
aws_access_key_id=credentials.credentials.access_key,
aws_secret_access_key=credentials.credentials.secret_key,
security_token=credentials.credentials.session_token)
def connect_to_aws(self, module, region):
connect_args = self.credentials
# only pass the profile name if it's set (as it is not supported by older boto versions)
if self.boto_profile:
connect_args['profile_name'] = self.boto_profile
self.boto_fix_security_token_in_profile(connect_args)
if self.iam_role:
sts_conn = sts.connect_to_region(region, **connect_args)
role = sts_conn.assume_role(self.iam_role, 'ansible_dynamic_inventory')
connect_args['aws_access_key_id'] = role.credentials.access_key
connect_args['aws_secret_access_key'] = role.credentials.secret_key
connect_args['security_token'] = role.credentials.session_token
conn = module.connect_to_region(region, **connect_args)
# connect_to_region will fail "silently" by returning None if the region name is wrong or not supported
if conn is None:
self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region)
return conn
def get_sts_token(request):
try:
key, secret, bucket_name = get_s3_creds()
policy_to_grant = {
'Statement': [{
'Action': ['s3:PutObject'],
'Effect': 'Allow',
'Resource': ['arn:aws:s3:::' + bucket_name + '/*']
}]
}
sts_conn = sts.connect_to_region(
'us-east-1', # edit and put the correct zone
aws_access_key_id=key,
aws_secret_access_key=secret)
token = sts_conn.get_federation_token(
name='iam_user', # The name of the federated user
duration=300,
policy=dumps(policy_to_grant))
return JsonResponse({'sts_token': token, 'status': 'ok'})
except:
return JsonResponse({'sts_token': None, 'status': 'error'})
def handle_security_credentials(role_name):
"""
Assumes an IAM role and returns the security credentials. Caches results in application context until expiration.
:param role_name: the IAM role to assume
"""
with app.app_context():
credentials = getattr(current_app, '_security_credentials', None)
expiration = getattr(current_app, '_security_credentials_expiration',
None)
if not credentials or datetime.now(pytz.utc) > expiration:
role = iam.connect_to_region(REGION).get_role(role_name)
session = sts.connect_to_region(REGION).assume_role(
role.arn, "Local")
credentials = jsonify({
"Code":
"Success",
"LastUpdated":
datetime.now(pytz.utc).strftime(DATE_FORMAT),
"Type":
"AWS-HMAC",
"AccessKeyId":
session.credentials.access_key,
"SecretAccessKey":
session.credentials.secret_key,
"Token":
session.credentials.session_token,
"Expiration":
session.credentials.expiration
})
current_app._security_credentials = credentials
current_app._security_credentials_expiration = datetime.strptime(
session.credentials.expiration,
DATE_FORMAT).replace(tzinfo=pytz.utc)
return credentials
def __init__(self, yamlFile):
self.logger = logging.getLogger(__name__)
# load the yaml file and turn it into a dict
thefile = open(yamlFile, 'r')
rendered_file = pystache.render(thefile.read(), dict(os.environ))
self.stackDict = yaml.safe_load(rendered_file)
# Make sure there is only one top level element in the yaml file
if len(self.stackDict.keys()) != 1:
error_message = ("Need one and only one mega stack name at the"
+ " top level, found %s")
self.logger.critical(error_message % len(self.stackDict.keys()))
exit(1)
# Now we know we only have one top element,
# that must be the mega stack name
self.name = self.stackDict.keys()[0]
# Find and set the mega stacks region. Exit if we can't find it
if 'region' in self.stackDict[self.name]:
self.region = self.stackDict[self.name]['region']
else:
self.logger.critical("No region specified for mega stack,"
+ " don't know where to build it.")
exit(1)
# Find and set the mega stack's AWS profile
if 'aws_profile' in self.stackDict[self.name]:
self.aws_profile = self.stackDict[self.name]['aws_profile']
else:
self.aws_profile = None
# Find and set the mega stack's STS role ARN
if 'sts_role' in self.stackDict[self.name]:
self.sts_role = self.stackDict[self.name]['sts_role']
else:
self.sts_role = None
# Connect to STS and assume the provided role
if self.sts_role is not None:
try:
stsconn = sts.connect_to_region(self.region,
profile_name=self.aws_profile)
role = stsconn.assume_role(role_arn=self.sts_role,
role_session_name='cumulus')
self.aws_access_key_id = role.credentials.access_key
self.aws_secret_access_key = role.credentials.secret_key
self.aws_session_token = role.credentials.session_token
self.logger.info("Using STS credentials to set up stack, stack"
+ " creation may fail if it takes longer than"
+ " 1 hour")
except BotoServerError as e:
self.logger.critical("Could not assume STS role")
self.logger.critical(e.message)
exit(1)
else:
self.aws_access_key_id = None
self.aws_secret_access_key = None
self.aws_session_token = None
# Connect to an AWS service using proper credentials
def connect(service):
kwargs = {}
if self.aws_access_key_id is not None \
and self.aws_secret_access_key is not None:
# Using an STS assumed role
kwargs['aws_access_key_id'] = self.aws_access_key_id
kwargs['aws_secret_access_key'] = self.aws_secret_access_key
kwargs['security_token'] = self.aws_session_token
elif self.aws_profile is not None:
# Using an AWS profile
kwargs['profile_name'] = self.aws_profile
return service.connect_to_region(self.region, **kwargs)
if 'account_id' in self.stackDict[self.name]:
# Get the account ID for the current AWS credentials
iamconn = connect(iam)
user_response = iamconn.get_user()['get_user_response']
user_result = user_response['get_user_result']
account_id = user_result['user']['arn'].split(':')[4]
# Check if the current account ID matches the stack's account ID
if account_id != str(self.stackDict[self.name]['account_id']):
self.logger.critical("Account ID of stack does not match the"
+ " account ID of your AWS credentials.")
exit(1)
self.sns_topic_arn = self.stackDict[self.name].get('sns-topic-arn', [])
if isinstance(self.sns_topic_arn, str):
self.sns_topic_arn = [self.sns_topic_arn]
for topic in self.sns_topic_arn:
if topic.split(':')[3] != self.region:
self.logger.critical("SNS Topic %s is not in the %s region."
% (topic, self.region))
exit(1)
self.global_tags = self.stackDict[self.name].get('tags', {})
# Array for holding CFStack objects once we create them
self.stack_objs = []
# Get the names of the sub stacks from the yaml file and sort in array
self.cf_stacks = self.stackDict[self.name]['stacks'].keys()
# Megastack holds the connection to CloudFormation and list of stacks
# currently in our region stops us making lots of calls to
# CloudFormation API for each stack
try:
self.cfconn = connect(cloudformation)
self.cf_desc_stacks = self._describe_all_stacks()
except boto.exception.NoAuthHandlerFound as exception:
self.logger.critical(
"No credentials found for connecting to CloudFormation: %s"
% exception)
exit(1)
# iterate through the stacks in the yaml file and create CFstack
# objects for them
for stack_name in self.cf_stacks:
the_stack = self.stackDict[self.name]['stacks'][stack_name]
if type(the_stack) is dict:
if the_stack.get('disable', False):
warn_message = ("Stack %s is disabled by configuration"
+ " directive. Skipping")
self.logger.warning(warn_message % stack_name)
continue
local_sns_arn = the_stack.get('sns-topic-arn',
self.sns_topic_arn)
if isinstance(local_sns_arn, str):
local_sns_arn = [local_sns_arn]
for topic in local_sns_arn:
if topic.split(':')[3] != self.region:
error_message = "SNS Topic %s is not in the %s region."
self.logger.critical(error_message
% (topic, self.region))
exit(1)
local_tags = the_stack.get('tags', {})
merged_tags = dict(self.global_tags.items()
+ local_tags.items())
# Add static cumulus-stack tag
merged_tags['cumulus-stack'] = self.name
if 'cf_template' in the_stack:
self.stack_objs.append(
CFStack(
mega_stack_name=self.name,
name=stack_name,
params=the_stack.get('params'),
template_name=the_stack['cf_template'],
cfconn=self.cfconn,
sns_topic_arn=local_sns_arn,
depends_on=the_stack.get('depends'),
tags=merged_tags
)
)
def sts_conn():
aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
return sts.connect_to_region('us-east-1',
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
