def delete_role_inline_policies(self, role_name):
"""Delete policies that inline to a role"""
try:
response = self._client.list_role_policies(RoleName=role_name)
if not response:
return False
for name in response.get('PolicyNames'):
Oprint.info(
'Deleting inline IAM policy {} for role {}'.format(
name, role_name), 'iam')
self._client.delete_role_policy(RoleName=role_name,
PolicyName=name)
Oprint.info(
'Inline IAM policy {} has been deleted'.format(name),
'iam')
except Exception as e:
Oprint.err(e.response['Error']['Message'], 'iam', exit=False)
return True
def create_policy(self,
policy_name,
policy_document,
delete=True,
**kwargs):
"""wrapper, check if policy exists before create"""
try:
Oprint.info('Creating IAM policy {}'.format(policy_name), 'iam')
policy = self.get_policy(policy_name=policy_name)
if policy and policy.get('Policy'):
if not delete:
Oprint.info(
'Found existing IAM policy {}'.format(policy_name),
'iam')
return policy
else:
# Can not delete a policy if it has been attached
if policy.get('Policy').get('AttachmentCount') > 0:
Oprint.warn(
'Policy {} already exists and has been attached to a role. Cannot delete'
.format(policy.get('Policy').get('PolicyName')),
'iam')
return policy
self._client.delete_policy(
PolicyArn=self.get_policy_arn(policy_name))
policy = self._client.create_policy(PolicyName=policy_name,
PolicyDocument=policy_document,
**kwargs)
Oprint.info('IAM policy {} has been created'.format(policy_name),
'iam')
except Exception as e:
Oprint.err(e, 'iam')
return policy
def delete_rule_targets(self, rule_name):
"""Delete all targes from a rule"""
try:
response = self._client.list_targets_by_rule(Rule=rule_name)
target_ids = [target['Id'] for target in response.get('Targets')]
d_response = self.delete_targets(rule_name=rule_name,
target_ids=target_ids)
if not d_response:
return False
while response.get('NextToken'):
response = self._client.list_targets_by_rule(
Rule=rule_name, NextToken=response.get('NextToken'))
target_ids = [
target['Id'] for target in response.get('Targets')
]
d_response = self.delete_targets(rule_name=rule_name,
target_ids=target_ids)
if not d_response:
return False
except Exception as e:
Oprint.err(e, self.NAME)
return True
def detach_role_managed_policies(self, role_name):
"""Detach managed policies that attache to a role"""
try:
response = self._client.list_attached_role_policies(
RoleName=role_name)
if not response:
return False
for policy in response.get('AttachedPolicies'):
Oprint.info(
'Detaching managed IAM policy {}'.format(
policy.get('PolicyName')), 'iam')
self._client.detach_role_policy(
RoleName=role_name, PolicyArn=policy.get('PolicyArn'))
Oprint.info(
'Managed IAM policy {} has been deteched'.format(
policy.get('PolicyName')), 'iam')
except Exception as e:
Oprint.err(e.response['Error']['Message'], 'iam', exit=False)
return True
def create_domain_name(self, domain_name, cert_name, cert_body, cert_private_key, cert_chain):
"""Create API custom domain name"""
try:
with open(cert_body, 'r') as outfile:
cert_body_str = outfile.read()
with open(cert_private_key, 'r') as outfile:
cert_private_key_str = outfile.read()
with open(cert_chain, 'r') as outfile:
cert_chain_str = outfile.read()
Oprint.info('Creating custom domain {}'.format(domain_name), 'apigateway')
response = self._client.create_domain_name(
domainName=domain_name,
certificateName=cert_name,
certificateBody=cert_body_str,
certificatePrivateKey=cert_private_key_str,
certificateChain=cert_chain_str
)
Oprint.info('Complete creating custom domain {}'.format(domain_name), 'apigateway')
except Exception as e:
Oprint.err(e, 'apigateway')
def get_current_venv_path(self):
"""
Returns the path to the current virtualenv
"""
Oprint.info('Identifying current virtualenv path', 'pip')
if 'VIRTUAL_ENV' in os.environ:
venv = os.environ['VIRTUAL_ENV']
elif os.path.exists('.python-version'): # pragma: no cover
try:
subprocess.check_output('pyenv', stderr=subprocess.STDOUT)
except OSError:
Oprint.err(
"This directory seems to have pyenv's local venv but pyenv executable was not found.",
'pip')
with open('.python-version', 'r') as f:
env_name = f.read()[:-1]
bin_path = subprocess.check_output(['pyenv', 'which',
'python']).decode('utf-8')
venv = bin_path[:bin_path.rfind(env_name)] + env_name
else: # pragma: no cover
Oprint.err("An active virtual environment is not found", 'lambda')
return venv
def excecute_change_set(self, change_set_name, stack_name, *args, **kwargs):
"""Run changeset to make it happen"""
try:
self.unlock_stack(stack_name=stack_name)
if self._args.get('-he') or self._args.get('--hide-event'):
waiter = CloudformationWaiterStackUpdate(self._client)
Oprint.info('Executing change set {} for updating stack {}'.format(change_set_name, stack_name), self.NAME)
response = self._client.execute_change_set(ChangeSetName=change_set_name, StackName=stack_name, *args, **kwargs)
waiter.wait(stack_name)
else:
Oprint.info('Executing change set {} for updating stack {}'.format(change_set_name, stack_name), self.NAME)
response = self._client.execute_change_set(ChangeSetName=change_set_name, StackName=stack_name, *args, **kwargs)
self.stack_events_waiter(stack_name=stack_name)
self.lock_stack(stack_name=stack_name)
except Exception as e:
Oprint.err(e, self.NAME)
self.verify_stack(mode='update', stack_id=stack_name)
return response
def create_stage_from_stage(self, from_stage, new_stage, api_name=None):
"""Create a new stage by given existing stage"""
# Get api_id
api = self.if_api_exist_by_name(api_name or self.get_apigateway_name())
if not api:
Oprint.err('API {} hasn\'t been created yet. Please create it first.'.format(api_name or self.get_apigateway_name()), 'apigateway')
# Get deployment ID from source stage
from_stage_info = self.get_stage(api.get('id'), from_stage)
if len(from_stage_info.get('deploymentId')) <= 0:
Oprint.warn('Stage: {} for API {} hasn\'t been deployed to internet yet, deploying...'.format(from_stage, api_name or self.get_apigateway_name()), 'apigateway')
deployment_id = self.create_deployment(api.get('id'), from_stage)
else:
deployment_id = from_stage_info.get('deploymentId')
to_stage_info = self.get_stage(api.get('id'), new_stage)
if to_stage_info:
# Delete new stage if exist
Oprint.warn('Stage {} exists for API {}. Removing it now...'.format(new_stage, api_name or self.get_apigateway_name()), 'apigateway')
self.delete_api_stage(api.get('id'), new_stage, api.get('name'))
# Create new stage
self.create_api_stage(api.get('id'), new_stage, deployment_id)
def sync(self):
"""Sync local asset to s3"""
if not self._config.get('AssetDirectory') or not self._config.get(
'AssetS3Bucket'):
Oprint.err(
'Your AssetDirectory or AssetS3Bucket is missing from {}'.
format(PROJECT_CONFIG_FILE), 's3')
# WEIRD!!!! isdir won't work without repr!!!!
if os.path.isdir(
repr('./{}'.format(self._config.get('AssetDirectory')))):
Oprint.err(
'Your asset directory {} doesn\'t exist'.format(
self._config.get('AssetDirectory')), 's3')
files = self.prepare_files_for_upload(
'./{}'.format(self._config.get('AssetDirectory')),
self._config.get('AssetDirectory'), S3_UPLOAD_EXCLUDE)
for f in files:
self.upload_file(self._config.get('AssetS3Bucket'),
f.get('path'),
f.get('key'),
ExtraArgs=f.get('extra_args'))
def prepare(self, templates, bucket=None):
"""Prepare all templates/validate/upload before create and update"""
if len(templates['children']) > 0 and not bucket:
Oprint.err('S3 bucket hasn\'t been provided for nested template', self.NAME)
# Validate syntax of the template
with open(templates['master'], 'r') as outfile:
self.validate_template(outfile.read())
for child_template in templates['children']:
with open(child_template, 'r') as outfile:
self.validate_template(outfile.read())
# If bucket provided, we upload
# all templates into the subfolder
if bucket:
path, template_name = os.path.split(templates['master'])
self._s3.upload_file(bucket, templates['master'], "{}/{}".format(self.get_name_id(), template_name))
for child_template in templates['children']:
path, template_name = os.path.split(child_template)
self._s3.upload_file(bucket, child_template, "{}/{}".format(self.get_name_id(), template_name))
return True
def update_stack(self, stack_name, capabilities=None, **kwargs):
"""Update a stack"""
try:
capabilities = capabilities or ['CAPABILITY_NAMED_IAM', 'CAPABILITY_IAM']
self.unlock_stack(stack_name=stack_name)
if self._args.get('-he') or self._args.get('--hide-event'):
waiter = CloudformationWaiterStackUpdate(self._client)
response = self._client.update_stack(
StackName=stack_name,
Capabilities=capabilities,
**kwargs
)
waiter.wait(stack_name)
else:
response = self._client.update_stack(
StackName=stack_name,
Capabilities=capabilities,
**kwargs
)
self.stack_events_waiter(stack_name=stack_name)
self.lock_stack(stack_name=stack_name)
except ClientError as ce:
if str(ce.response['Error']['Message']) == 'No updates are to be performed.':
Oprint.warn('AWS Validation Error: {} (If purely there isn\'t any changes to your cloudformation, you can safely ignore it)'.format(str(ce.response['Error']['Message'])), self.NAME)
return True
else:
Oprint.err(str(ce.response['Error']['Message']), self.NAME)
except Exception as e:
Oprint.err(e, self.NAME)
return False
self.verify_stack(mode='update', stack_id=response.get('StackId'))
return True
def generate_logs(self, log_group_name):
"""Log generator"""
if not log_group_name:
Oprint.err('No log group name given, exit...', 'logs')
kw = {}
kw['log_group_name'] = log_group_name
kw['interleaved'] = True
start_time = self.get_start_time()
if start_time:
kw['startTime'] = start_time
end_time = self.get_end_time()
if end_time:
kw['endTime'] = end_time
while True:
try:
response = self.filter_logs(**kw)
except Exception as e:
Oprint.err(
'No Cloudwatch logs found for {}'.format(log_group_name),
'logs')
if response:
for event in response.get('events'):
yield event
if response.get('nextToken'):
kw['nextToken'] = response.get('nextToken')
else:
yield self._wait
else:
Oprint.err(
'No Cloudwatch logs found for {}'.format(log_group_name),
'logs')
def create_lambda_role(self, role_name, role_policy):
"""
Create role for Lambda, all policies
store in the assume role policy doc
so that we can do update
If not set, will using default
which enables Lambda for invoking
and logging
"""
try:
Oprint.info(
'Start creating role {} and policie for Lambda'.format(
role_name), 'iam')
assume_roles = []
if role_policy:
assume_roles = role_policy.get('AssumeRoles')
role_doc = self.get_lambda_default_assume_role_doc(
extra_services=assume_roles)
role = self.get_role(role_name)
if not role:
role = self._client.create_role(
RoleName=role_name,
AssumeRolePolicyDocument=json.dumps(role_doc))
else:
self._client.update_assume_role_policy(
RoleName=role_name, PolicyDocument=json.dumps(role_doc))
to_replace = {
"$region": self.get_region(),
"$accountId": self.get_account_id()
}
policy_doc = []
if role_policy and role_policy.get('PolicyDocument'):
_, policy_doc = FileLoader(
file_path=role_policy.get('PolicyDocument')).process()
policy_doc = policy_doc['Statement']
policy_doc = self.get_lambda_default_policy_doc(
extra_statement=policy_doc)
policy_doc = update_template(json.dumps(policy_doc), to_replace)
# Do inline policy so that when deleting the role, it'll be deleted
self._client.put_role_policy(
RoleName=role_name,
PolicyName='{}-policy'.format(role_name),
PolicyDocument=policy_doc)
if role_policy and role_policy is dict and role_policy.get(
'ManagedPolicyArns'):
for m_policy_arn in role_policy.get('ManagedPolicyArns'):
self._client.attach_role_policy(RoleName=role_name,
PolicyArn=m_policy_arn)
Oprint.info(
'Complete creating role {} and policie for Lambda'.format(
role_name), 'iam')
except Exception as e:
Oprint.err(e, 'iam')
return role
def package_install(self, tmp_path):
"""Install requirement"""
if os.path.isfile('{}/{}'.format(
tmp_path,
os.getenv('PIP_REQUIREMENTS_FILE', PIP_REQUIREMENTS_FILE))):
with open('{}/{}'.format(
tmp_path,
os.getenv('PIP_REQUIREMENTS_FILE',
PIP_REQUIREMENTS_FILE))) as f:
requirements = [
item.strip().lower() for item in f.read().splitlines()
if item.strip()
]
try:
lambda_pkg_to_install = {}
# Filter function to find package should
# be fetched from lambda package
def find_lambda_pkg(item):
found = False
for name, detail in lambda_packages.items():
if item.startswith(name.lower()):
lambda_pkg_to_install[name.lower()] = detail
return False
else:
continue
return True
requirements = filter(find_lambda_pkg, requirements)
# always install setup tool
requirements.append('setuptools')
for name, detail in lambda_pkg_to_install.iteritems():
Oprint.info(
'Installing Amazon Linux AMI bianry package {} to {}'.
format(name, tmp_path), 'pip')
tar = tarfile.open(detail['path'], mode="r:gz")
for member in tar.getmembers():
if member.isdir():
shutil.rmtree(os.path.join(tmp_path, member.name),
ignore_errors=True)
continue
#tar.extract(member, os.getenv('PIP_VENDOR_FOLDER', PIP_VENDOR_FOLDER))
tar.extract(member, tmp_path)
tmp_requirements = tempfile.NamedTemporaryFile(delete=False)
for line in requirements:
tmp_requirements.write(line + '\n')
tmp_requirements.close()
Oprint.info(
'Installing python package dependancies to {}'.format(
tmp_path), 'pip')
spinner.start()
os.system('pip install -t {} -r {} &>/dev/null'.format(
tmp_path, tmp_requirements.name))
spinner.stop()
except Exception as e:
spinner.stop()
Oprint.err(e, 'pip')
else:
Oprint.warn(
'{} could not be found, no dependencies will be installed'.
format(
os.getenv('PIP_REQUIREMENTS_FILE',
PIP_REQUIREMENTS_FILE)), 'pip')
def get_zipped_package(self, function_config):
"""Packaging lambda"""
func_name = function_config.get('FunctionName')
func_type = function_config.get('Type')
# Create packaging temp dir
lambda_temp_dir = tempfile.mkdtemp()
# Create zip file temp dir
target_temp_dir = tempfile.mkdtemp()
target = '{}/{}'.format(target_temp_dir, self.get_zip_name(func_name))
self.add_init_file_to_root(lambda_temp_dir)
if func_type == self.FUNCTION_TYPE_WSGI:
self.pip_wsgi_install(lambda_temp_dir)
# Heater is one file only from lmdo. don't need packages
if func_type != self.FUNCTION_TYPE_HEATER:
# Go only need executables
if func_type == self.FUNCTION_TYPE_GO:
if not function_config.get('ExecutableName'):
Oprint.err(
'ExecutableName is not defined in lmdo config, function {} won\'t be deployed'
.format(func_name), self.NAME)
return False, False
# We only have on executable needed
shutil.copy(
os.path.join(os.getcwd(),
function_config.get('ExecutableName')),
lambda_temp_dir)
else:
# Copy project files
copytree(os.getcwd(),
lambda_temp_dir,
ignore=shutil.ignore_patterns('*.git*'))
# Installing package
self.dependency_packaging(lambda_temp_dir)
replace_path = [{'from_path': lambda_temp_dir, 'to_path': '.'}]
# Zip what we've got so far
zipper(lambda_temp_dir, target, LAMBDA_EXCLUDE, False,
replace_path)
# Default type function doesn't need lmdo's lambda wrappers
if func_type != self.FUNCTION_TYPE_DEFAULT:
# Don't load lmdo __init__.py
if LAMBDA_EXCLUDE.get('LAMBDA_EXCLUDE'):
LAMBDA_EXCLUDE['file_with_path'].append(
'*{}/{}/__init__.py'.format(self.LMDO_HANDLER_DIR,
func_type))
else:
LAMBDA_EXCLUDE['file_with_path'] = [
'*{}/{}/__init__.py'.format(self.LMDO_HANDLER_DIR,
func_type)
]
replace_path = [{
'from_path': self.get_lmdo_function_dir(func_type),
'to_path': '.'
}]
# Zip extra lmdo function handler
zipper(self.get_lmdo_function_dir(func_type), target,
LAMBDA_EXCLUDE, False, replace_path)
shutil.rmtree(lambda_temp_dir)
return (target_temp_dir, target)
def sys_pause(message, match):
"""pause program to catch user input"""
name = raw_input(message)
if not fnmatch.fnmatch(name, match):
Oprint.err('Exit excecution', 'lmdo')
