def delete_role_inline_policies(self, role_name): """Delete policies that inline to a role""" try: response = self._client.list_role_policies(RoleName=role_name) if not response: return False for name in response.get('PolicyNames'): Oprint.info( 'Deleting inline IAM policy {} for role {}'.format( name, role_name), 'iam') self._client.delete_role_policy(RoleName=role_name, PolicyName=name) Oprint.info( 'Inline IAM policy {} has been deleted'.format(name), 'iam') except Exception as e: Oprint.err(e.response['Error']['Message'], 'iam', exit=False) return True
def create_policy(self, policy_name, policy_document, delete=True, **kwargs): """wrapper, check if policy exists before create""" try: Oprint.info('Creating IAM policy {}'.format(policy_name), 'iam') policy = self.get_policy(policy_name=policy_name) if policy and policy.get('Policy'): if not delete: Oprint.info( 'Found existing IAM policy {}'.format(policy_name), 'iam') return policy else: # Can not delete a policy if it has been attached if policy.get('Policy').get('AttachmentCount') > 0: Oprint.warn( 'Policy {} already exists and has been attached to a role. Cannot delete' .format(policy.get('Policy').get('PolicyName')), 'iam') return policy self._client.delete_policy( PolicyArn=self.get_policy_arn(policy_name)) policy = self._client.create_policy(PolicyName=policy_name, PolicyDocument=policy_document, **kwargs) Oprint.info('IAM policy {} has been created'.format(policy_name), 'iam') except Exception as e: Oprint.err(e, 'iam') return policy
def delete_rule_targets(self, rule_name): """Delete all targes from a rule""" try: response = self._client.list_targets_by_rule(Rule=rule_name) target_ids = [target['Id'] for target in response.get('Targets')] d_response = self.delete_targets(rule_name=rule_name, target_ids=target_ids) if not d_response: return False while response.get('NextToken'): response = self._client.list_targets_by_rule( Rule=rule_name, NextToken=response.get('NextToken')) target_ids = [ target['Id'] for target in response.get('Targets') ] d_response = self.delete_targets(rule_name=rule_name, target_ids=target_ids) if not d_response: return False except Exception as e: Oprint.err(e, self.NAME) return True
def detach_role_managed_policies(self, role_name): """Detach managed policies that attache to a role""" try: response = self._client.list_attached_role_policies( RoleName=role_name) if not response: return False for policy in response.get('AttachedPolicies'): Oprint.info( 'Detaching managed IAM policy {}'.format( policy.get('PolicyName')), 'iam') self._client.detach_role_policy( RoleName=role_name, PolicyArn=policy.get('PolicyArn')) Oprint.info( 'Managed IAM policy {} has been deteched'.format( policy.get('PolicyName')), 'iam') except Exception as e: Oprint.err(e.response['Error']['Message'], 'iam', exit=False) return True
def create_domain_name(self, domain_name, cert_name, cert_body, cert_private_key, cert_chain): """Create API custom domain name""" try: with open(cert_body, 'r') as outfile: cert_body_str = outfile.read() with open(cert_private_key, 'r') as outfile: cert_private_key_str = outfile.read() with open(cert_chain, 'r') as outfile: cert_chain_str = outfile.read() Oprint.info('Creating custom domain {}'.format(domain_name), 'apigateway') response = self._client.create_domain_name( domainName=domain_name, certificateName=cert_name, certificateBody=cert_body_str, certificatePrivateKey=cert_private_key_str, certificateChain=cert_chain_str ) Oprint.info('Complete creating custom domain {}'.format(domain_name), 'apigateway') except Exception as e: Oprint.err(e, 'apigateway')
def get_current_venv_path(self): """ Returns the path to the current virtualenv """ Oprint.info('Identifying current virtualenv path', 'pip') if 'VIRTUAL_ENV' in os.environ: venv = os.environ['VIRTUAL_ENV'] elif os.path.exists('.python-version'): # pragma: no cover try: subprocess.check_output('pyenv', stderr=subprocess.STDOUT) except OSError: Oprint.err( "This directory seems to have pyenv's local venv but pyenv executable was not found.", 'pip') with open('.python-version', 'r') as f: env_name = f.read()[:-1] bin_path = subprocess.check_output(['pyenv', 'which', 'python']).decode('utf-8') venv = bin_path[:bin_path.rfind(env_name)] + env_name else: # pragma: no cover Oprint.err("An active virtual environment is not found", 'lambda') return venv
def excecute_change_set(self, change_set_name, stack_name, *args, **kwargs): """Run changeset to make it happen""" try: self.unlock_stack(stack_name=stack_name) if self._args.get('-he') or self._args.get('--hide-event'): waiter = CloudformationWaiterStackUpdate(self._client) Oprint.info('Executing change set {} for updating stack {}'.format(change_set_name, stack_name), self.NAME) response = self._client.execute_change_set(ChangeSetName=change_set_name, StackName=stack_name, *args, **kwargs) waiter.wait(stack_name) else: Oprint.info('Executing change set {} for updating stack {}'.format(change_set_name, stack_name), self.NAME) response = self._client.execute_change_set(ChangeSetName=change_set_name, StackName=stack_name, *args, **kwargs) self.stack_events_waiter(stack_name=stack_name) self.lock_stack(stack_name=stack_name) except Exception as e: Oprint.err(e, self.NAME) self.verify_stack(mode='update', stack_id=stack_name) return response
def create_stage_from_stage(self, from_stage, new_stage, api_name=None): """Create a new stage by given existing stage""" # Get api_id api = self.if_api_exist_by_name(api_name or self.get_apigateway_name()) if not api: Oprint.err('API {} hasn\'t been created yet. Please create it first.'.format(api_name or self.get_apigateway_name()), 'apigateway') # Get deployment ID from source stage from_stage_info = self.get_stage(api.get('id'), from_stage) if len(from_stage_info.get('deploymentId')) <= 0: Oprint.warn('Stage: {} for API {} hasn\'t been deployed to internet yet, deploying...'.format(from_stage, api_name or self.get_apigateway_name()), 'apigateway') deployment_id = self.create_deployment(api.get('id'), from_stage) else: deployment_id = from_stage_info.get('deploymentId') to_stage_info = self.get_stage(api.get('id'), new_stage) if to_stage_info: # Delete new stage if exist Oprint.warn('Stage {} exists for API {}. Removing it now...'.format(new_stage, api_name or self.get_apigateway_name()), 'apigateway') self.delete_api_stage(api.get('id'), new_stage, api.get('name')) # Create new stage self.create_api_stage(api.get('id'), new_stage, deployment_id)
def sync(self): """Sync local asset to s3""" if not self._config.get('AssetDirectory') or not self._config.get( 'AssetS3Bucket'): Oprint.err( 'Your AssetDirectory or AssetS3Bucket is missing from {}'. format(PROJECT_CONFIG_FILE), 's3') # WEIRD!!!! isdir won't work without repr!!!! if os.path.isdir( repr('./{}'.format(self._config.get('AssetDirectory')))): Oprint.err( 'Your asset directory {} doesn\'t exist'.format( self._config.get('AssetDirectory')), 's3') files = self.prepare_files_for_upload( './{}'.format(self._config.get('AssetDirectory')), self._config.get('AssetDirectory'), S3_UPLOAD_EXCLUDE) for f in files: self.upload_file(self._config.get('AssetS3Bucket'), f.get('path'), f.get('key'), ExtraArgs=f.get('extra_args'))
def prepare(self, templates, bucket=None): """Prepare all templates/validate/upload before create and update""" if len(templates['children']) > 0 and not bucket: Oprint.err('S3 bucket hasn\'t been provided for nested template', self.NAME) # Validate syntax of the template with open(templates['master'], 'r') as outfile: self.validate_template(outfile.read()) for child_template in templates['children']: with open(child_template, 'r') as outfile: self.validate_template(outfile.read()) # If bucket provided, we upload # all templates into the subfolder if bucket: path, template_name = os.path.split(templates['master']) self._s3.upload_file(bucket, templates['master'], "{}/{}".format(self.get_name_id(), template_name)) for child_template in templates['children']: path, template_name = os.path.split(child_template) self._s3.upload_file(bucket, child_template, "{}/{}".format(self.get_name_id(), template_name)) return True
def update_stack(self, stack_name, capabilities=None, **kwargs): """Update a stack""" try: capabilities = capabilities or ['CAPABILITY_NAMED_IAM', 'CAPABILITY_IAM'] self.unlock_stack(stack_name=stack_name) if self._args.get('-he') or self._args.get('--hide-event'): waiter = CloudformationWaiterStackUpdate(self._client) response = self._client.update_stack( StackName=stack_name, Capabilities=capabilities, **kwargs ) waiter.wait(stack_name) else: response = self._client.update_stack( StackName=stack_name, Capabilities=capabilities, **kwargs ) self.stack_events_waiter(stack_name=stack_name) self.lock_stack(stack_name=stack_name) except ClientError as ce: if str(ce.response['Error']['Message']) == 'No updates are to be performed.': Oprint.warn('AWS Validation Error: {} (If purely there isn\'t any changes to your cloudformation, you can safely ignore it)'.format(str(ce.response['Error']['Message'])), self.NAME) return True else: Oprint.err(str(ce.response['Error']['Message']), self.NAME) except Exception as e: Oprint.err(e, self.NAME) return False self.verify_stack(mode='update', stack_id=response.get('StackId')) return True
def generate_logs(self, log_group_name): """Log generator""" if not log_group_name: Oprint.err('No log group name given, exit...', 'logs') kw = {} kw['log_group_name'] = log_group_name kw['interleaved'] = True start_time = self.get_start_time() if start_time: kw['startTime'] = start_time end_time = self.get_end_time() if end_time: kw['endTime'] = end_time while True: try: response = self.filter_logs(**kw) except Exception as e: Oprint.err( 'No Cloudwatch logs found for {}'.format(log_group_name), 'logs') if response: for event in response.get('events'): yield event if response.get('nextToken'): kw['nextToken'] = response.get('nextToken') else: yield self._wait else: Oprint.err( 'No Cloudwatch logs found for {}'.format(log_group_name), 'logs')
def create_lambda_role(self, role_name, role_policy): """ Create role for Lambda, all policies store in the assume role policy doc so that we can do update If not set, will using default which enables Lambda for invoking and logging """ try: Oprint.info( 'Start creating role {} and policie for Lambda'.format( role_name), 'iam') assume_roles = [] if role_policy: assume_roles = role_policy.get('AssumeRoles') role_doc = self.get_lambda_default_assume_role_doc( extra_services=assume_roles) role = self.get_role(role_name) if not role: role = self._client.create_role( RoleName=role_name, AssumeRolePolicyDocument=json.dumps(role_doc)) else: self._client.update_assume_role_policy( RoleName=role_name, PolicyDocument=json.dumps(role_doc)) to_replace = { "$region": self.get_region(), "$accountId": self.get_account_id() } policy_doc = [] if role_policy and role_policy.get('PolicyDocument'): _, policy_doc = FileLoader( file_path=role_policy.get('PolicyDocument')).process() policy_doc = policy_doc['Statement'] policy_doc = self.get_lambda_default_policy_doc( extra_statement=policy_doc) policy_doc = update_template(json.dumps(policy_doc), to_replace) # Do inline policy so that when deleting the role, it'll be deleted self._client.put_role_policy( RoleName=role_name, PolicyName='{}-policy'.format(role_name), PolicyDocument=policy_doc) if role_policy and role_policy is dict and role_policy.get( 'ManagedPolicyArns'): for m_policy_arn in role_policy.get('ManagedPolicyArns'): self._client.attach_role_policy(RoleName=role_name, PolicyArn=m_policy_arn) Oprint.info( 'Complete creating role {} and policie for Lambda'.format( role_name), 'iam') except Exception as e: Oprint.err(e, 'iam') return role
def package_install(self, tmp_path): """Install requirement""" if os.path.isfile('{}/{}'.format( tmp_path, os.getenv('PIP_REQUIREMENTS_FILE', PIP_REQUIREMENTS_FILE))): with open('{}/{}'.format( tmp_path, os.getenv('PIP_REQUIREMENTS_FILE', PIP_REQUIREMENTS_FILE))) as f: requirements = [ item.strip().lower() for item in f.read().splitlines() if item.strip() ] try: lambda_pkg_to_install = {} # Filter function to find package should # be fetched from lambda package def find_lambda_pkg(item): found = False for name, detail in lambda_packages.items(): if item.startswith(name.lower()): lambda_pkg_to_install[name.lower()] = detail return False else: continue return True requirements = filter(find_lambda_pkg, requirements) # always install setup tool requirements.append('setuptools') for name, detail in lambda_pkg_to_install.iteritems(): Oprint.info( 'Installing Amazon Linux AMI bianry package {} to {}'. format(name, tmp_path), 'pip') tar = tarfile.open(detail['path'], mode="r:gz") for member in tar.getmembers(): if member.isdir(): shutil.rmtree(os.path.join(tmp_path, member.name), ignore_errors=True) continue #tar.extract(member, os.getenv('PIP_VENDOR_FOLDER', PIP_VENDOR_FOLDER)) tar.extract(member, tmp_path) tmp_requirements = tempfile.NamedTemporaryFile(delete=False) for line in requirements: tmp_requirements.write(line + '\n') tmp_requirements.close() Oprint.info( 'Installing python package dependancies to {}'.format( tmp_path), 'pip') spinner.start() os.system('pip install -t {} -r {} &>/dev/null'.format( tmp_path, tmp_requirements.name)) spinner.stop() except Exception as e: spinner.stop() Oprint.err(e, 'pip') else: Oprint.warn( '{} could not be found, no dependencies will be installed'. format( os.getenv('PIP_REQUIREMENTS_FILE', PIP_REQUIREMENTS_FILE)), 'pip')
def get_zipped_package(self, function_config): """Packaging lambda""" func_name = function_config.get('FunctionName') func_type = function_config.get('Type') # Create packaging temp dir lambda_temp_dir = tempfile.mkdtemp() # Create zip file temp dir target_temp_dir = tempfile.mkdtemp() target = '{}/{}'.format(target_temp_dir, self.get_zip_name(func_name)) self.add_init_file_to_root(lambda_temp_dir) if func_type == self.FUNCTION_TYPE_WSGI: self.pip_wsgi_install(lambda_temp_dir) # Heater is one file only from lmdo. don't need packages if func_type != self.FUNCTION_TYPE_HEATER: # Go only need executables if func_type == self.FUNCTION_TYPE_GO: if not function_config.get('ExecutableName'): Oprint.err( 'ExecutableName is not defined in lmdo config, function {} won\'t be deployed' .format(func_name), self.NAME) return False, False # We only have on executable needed shutil.copy( os.path.join(os.getcwd(), function_config.get('ExecutableName')), lambda_temp_dir) else: # Copy project files copytree(os.getcwd(), lambda_temp_dir, ignore=shutil.ignore_patterns('*.git*')) # Installing package self.dependency_packaging(lambda_temp_dir) replace_path = [{'from_path': lambda_temp_dir, 'to_path': '.'}] # Zip what we've got so far zipper(lambda_temp_dir, target, LAMBDA_EXCLUDE, False, replace_path) # Default type function doesn't need lmdo's lambda wrappers if func_type != self.FUNCTION_TYPE_DEFAULT: # Don't load lmdo __init__.py if LAMBDA_EXCLUDE.get('LAMBDA_EXCLUDE'): LAMBDA_EXCLUDE['file_with_path'].append( '*{}/{}/__init__.py'.format(self.LMDO_HANDLER_DIR, func_type)) else: LAMBDA_EXCLUDE['file_with_path'] = [ '*{}/{}/__init__.py'.format(self.LMDO_HANDLER_DIR, func_type) ] replace_path = [{ 'from_path': self.get_lmdo_function_dir(func_type), 'to_path': '.' }] # Zip extra lmdo function handler zipper(self.get_lmdo_function_dir(func_type), target, LAMBDA_EXCLUDE, False, replace_path) shutil.rmtree(lambda_temp_dir) return (target_temp_dir, target)
def sys_pause(message, match): """pause program to catch user input""" name = raw_input(message) if not fnmatch.fnmatch(name, match): Oprint.err('Exit excecution', 'lmdo')