def upload_file(self, bucket_name, file_path, key, **kwargs): """Upload file to S3, provide network progress bar""" # Check if bucket exist, create one if user agrees if not self.if_bucket_exist(bucket_name): sys_pause( 'Bucket {} doesn\'t exist! Do you want to create it? [yes/no]'. format(bucket_name), 'yes') self.create_bucket(bucket_name) file_size = os.path.getsize(file_path) / 1000000 if round(file_size) <= 0: file_size = 'size:{}B'.format(os.path.getsize(file_path)) else: file_size = 'size:{}MB'.format(file_size) Oprint.info( 'Start uploading {} to S3 bucket {}. ({})'.format( key, bucket_name, file_size), 's3') #waiter = S3WaiterObjectCreate(self._client) self._client.upload_file(file_path, bucket_name, key, Callback=FileUploadProgress(file_path), **kwargs) #waiter.wait(bucket_name, key) Oprint.info('Complete uploading {}. ({})'.format(key, file_size), 's3') return True
def get_stack_output(self, stack_name, key): """ Duplicate function to cloudformation due to avoiding circular condition as cloudformation import this module as well """ try: if not self._stack_info_cache.get(stack_name): self._stack_info_cache[stack_name] = AWSBase().get_client( 'cloudformation').describe_stacks(StackName=stack_name) outputs = self._stack_info_cache[stack_name]['Stacks'][0][ 'Outputs'] for opts in outputs: if opts['OutputKey'] == key: return opts['OutputValue'] Oprint.warn( 'Cannot find key {} output from stack {}'.format( key, stack_name), 'lmdo') except Exception: Oprint.warn( 'Error while retrieving output from {}'.format(stack_name), 'lmdo') return None
def execute(self): Oprint.info('Start deploying service', 'lmdo') self._dispatcher.run(CreateCommand(self._cloudformation)) self._dispatcher.run(CreateCommand(self._lambda)) self._dispatcher.run(CreateCommand(self._apigateway)) self._dispatcher.run(CreateCommand(self._cloudwatchevent)) Oprint.info('Complete deploying service', 'lmdo')
def create_wsgi_api(self): """Create/Update api definition for wsgi app""" swagger_api = False # If there is already an exsiting swagger api template, # fetch it so we won't duplicate it #if os.path.isfile(self.get_swagger_template()) and self.get_apigateway_name(): swagger_api = self.if_api_exist_by_name(self.get_apigateway_name()) iam = IAM() for lm_func in self._config.get('Lambda'): if lm_func.get('Type') != AWSLambda.FUNCTION_TYPE_WSGI or lm_func.get('DisableApiGateway'): continue function_name = self.get_lmdo_format_name(lm_func.get('FunctionName')) role = iam.get_lambda_apigateway_default_role(function_name) Oprint.info('Create/Update API Gateway for wsgi function {}'.format(lm_func.get('FunctionName')), 'apigateway') to_replace = { "$title": self.get_apigateway_name(), "$version": str(datetime.datetime.utcnow()), "$basePath": lm_func.get('ApiBasePath') or '/res', "$apiRegion": self.get_region(), "$functionRegion": self.get_region(), "$accountId": self.get_account_id(), "$functionName": function_name, "$credentials": role['Role'].get('Arn') } # Enable cognito user pool as authorizer if lm_func.get('CognitoUserPoolId'): se_replace = { "$apiRegion": self.get_region(), "$accountId": self.get_account_id(), "$userPoolId": lm_func.get('CognitoUserPoolId'), "$CognitoUserPool": 'CognitoUserPool-{}'.format(lm_func.get('FunctionName')) } to_replace["$securityDefinitions"] = self.get_apigateway_authorizer(se_replace) to_replace["$authorizer"] = '{"' + str(se_replace['$CognitoUserPool'])+'":[]}' else: to_replace["$securityDefinitions"] = '' to_replace["$authorizer"] = '' template_dir = get_template(APIGATEWAY_SWAGGER_WSGI) if not template_dir: Oprint.err('Template {} for creating wsgi APIGateway hasn\'t been installed or missing'.format(APIGATEWAY_SWAGGER_WSGI), 'apigateway') with open(template_dir, 'r') as outfile: body = update_template(outfile.read(), to_replace) if not swagger_api: swagger_api = self.import_rest_api(body) else: # Always overwrite for update self.put_rest_api(swagger_api.get('id'), body, 'merge') return swagger_api
def create_stack(self, stack_name, capabilities=None, **kwargs): """Create stack""" try: capabilities = capabilities or ['CAPABILITY_NAMED_IAM', 'CAPABILITY_IAM'] if self._args.get('-he') or self._args.get('--hide-event'): waiter = CloudformationWaiterStackCreate(self._client) response = self._client.create_stack( StackName=stack_name, Capabilities=capabilities, **kwargs ) waiter.wait(stack_name) else: response = self._client.create_stack( StackName=stack_name, Capabilities=capabilities, **kwargs ) self.stack_events_waiter(stack_name=stack_name) self.lock_stack(stack_name=stack_name) except Exception as e: Oprint.err(e, self.NAME) return False self.verify_stack(mode='create', stack_id=response.get('StackId')) return True
def delete_stack(self, stack_name, no_policy=False): """Remove a stack by given name""" # Don't do anything if doesn't exist stack_info = self.get_stack(stack_name=stack_name) if not stack_info: return True try: if not no_policy: self.unlock_stack(stack_name=stack_name) if self._args.get('-he') or self._args.get('--hide-event'): waiter = CloudformationWaiterStackDelete(self._client) response = self._client.delete_stack(StackName=stack_name) waiter.wait(stack_name) else: response = self._client.delete_stack(StackName=stack_name) self.stack_events_waiter(stack_name=stack_name) except Exception as e: Oprint.err(e, self.NAME) return False self.verify_stack(mode='delete', stack_id=stack_info['Stacks'][0]['StackId']) return True
def init(self): """Initiating the project and provide a sample lmdo.yaml file""" if self._args.get('<project_name>'): mkdir('./{}'.format(self._args.get('<project_name>'))) """Copy lmdo.yaml over""" # Do not copy over unless it's a clearn dir if os.path.isfile( os.path.join(self._args.get('<project_name>'), PROJECT_CONFIG_FILE)): Oprint.err( 'Your have existing {} already, exiting...'.format( PROJECT_CONFIG_FILE), 'lmdo') pkg_dir = self.get_installed_path() if pkg_dir: copytree(os.path.join(pkg_dir, 'template'), './{}'.format(self._args.get('<project_name>'))) elif self._args.get('config'): pkg_dir = self.get_installed_path() # Don't override existing lmdo.yaml if os.path.isfile(PROJECT_CONFIG_FILE): Oprint.warn( 'You have existing {} file, a copy will be created with name {}.copy' .format(PROJECT_CONFIG_FILE, PROJECT_CONFIG_FILE), 'lmdo') shutil.copyfile( os.path.join(pkg_dir, 'template', PROJECT_CONFIG_FILE), '{}.copy'.format(PROJECT_CONFIG_FILE)) else: shutil.copyfile( os.path.join(pkg_dir, 'template', PROJECT_CONFIG_FILE), PROJECT_CONFIG_FILE)
def validate_template(self, template_body): """Validate template via content""" try: result = self._client.validate_template(TemplateBody=template_body) except Exception as e: Oprint.err(e, self.NAME) return True
def validate(self): """ Set default value and check if mandatory keys exist """ # Set default profile if doesn't exist if not self.get('Profile'): self._config['Profile'] = 'default' # Set default user if doesn't exist if not self.get('User'): self._config['User'] = '******' # Set default service if doesn't exist if not self.get('Service'): self._config['Service'] = 'default' # Set default stage if doesn't exist if not self.get('Stage'): self._config['Stage'] = 'dev' # Check if all keys available for key in CONFIG_MANDATORY_KEYS: if key not in self._config: Oprint.err('{} is missing from config file'.format(key), 'lmdo')
def loading_strategy(self): """ Load file into json object Returns a tuple raw and json """ try: if not self.file_allowed(): raise Exception('File type {} is not allowed'.format( self.get_ext())) with open(self._file_path, 'r') as outfile: raw = outfile.read() if self.isYaml(): if self._yaml_replacements: for key, value in self._yaml_replacements.iteritems(): raw = raw.replace(key, value) json_content = FileLoader.toJson(raw, file_name=self._file_path) return raw, json_content except Exception as e: Oprint.err(e) else: raise Exception('File type {} is not allowed'.format( self.get_ext()))
def execute(self): Oprint.info('Start tear down service', 'lmdo') self._dispatcher.run(DeleteCommand(self._apigateway)) self._dispatcher.run(DeleteCommand(self._lambda)) self._dispatcher.run(DeleteCommand(self._cloudformation)) self._dispatcher.run(DeleteCommand(self._cloudwatchevent)) Oprint.info('Service has been destroy', 'lmdo')
def create_apigateway_lambda_role(self, role_name): """Create APIGateway role that can invoke lambda""" try: response = self.get_role(role_name) if response: Oprint.warn( 'Role {} exists, no action required'.format(role_name), 'iam') template = get_template(IAM_ROLE_APIGATEWAY_LAMBDA) if not template: return False if not response: with open(template, 'r') as outfile: assume_policy = outfile.read() response = self.create_role(role_name, assume_policy) policy = self.create_default_policy( role_name, self.create_policy_name(role_name, 'lambda-invoke'), IAM_POLICY_APIGATEWAY_LAMBDA_INVOKE) except Exception as e: Oprint.err(e, 'iam') return response
def create_default_events_role(self, role_name): """Create lmdo default event role""" try: response = self.get_role(role_name) if response: Oprint.warn( 'Role {} exists, no action required'.format(role_name), 'iam') return response template = get_template(IAM_ROLE_EVENTS) if not template: return False if not response: with open(template, 'r') as outfile: assume_policy = outfile.read() response = self.create_role(role_name, assume_policy) policy = self.create_default_policy( role_name, self.create_policy_name(role_name, 'default-events'), IAM_POLICY_EVENTS) except Exception as e: Oprint.err(e, 'apigateway') return response
def if_api_exist_by_name(self, api_name): """ Check if an API exist by given name Note: APIGateway allows name to be duplicated, it uses internal ID to identify the actual APIs. We won't consider using same name for APIs, hence return the first item which should be unique """ try: limit = 100 found = False pos = None while True: if pos: response = self._client.get_rest_apis(position=pos, limit=limit) else: response = self._client.get_rest_apis(limit=limit) pos = response.get('position') if len(response.get('items')) > 0: for api in response.get('items'): if api_name == api.get('name'): found = api break if not pos: break return found except Exception as e: Oprint.err(e, 'apigateway') return False
def describe_change_set(self, change_set_name, *args, **kwargs): """Get change set information""" try: response = self._client.describe_change_set(ChangeSetName=change_set_name, *args, **kwargs) except Exception as e: Oprint.err(e, self.NAME) return response
def list_existing_change_set(self, stack_name, *args, **kwargs): """List all existing stack""" try: response = self._client.list_change_sets(StackName=stack_name, *args, **kwargs) except Exception as e: Oprint.warn(e, self.NAME) return response
def create(self): """Create/Update stack""" # Don't run if we don't have templates if not self._config.get('CloudFormation'): Oprint.info('No cloudformation found, skip', self.NAME) return True self.process()
def delete_change_set(self, change_set_name, *args, **kwargs): """Delete existing change set""" try: response = self._client.delete_change_set(ChangeSetName=change_set_name, *args, **kwargs) except Exception as e: Oprint.warn(e, self.NAME) return True
def list_functions(self, **kwargs): """Wrapper for listing functions""" try: response = self._client.list_functions(**kwargs) except Exception as e: Oprint.err(e, 'lambda') return response
def wait(self, change_set_name, stack_name): try: Oprint.info('Start creating change set {} for stack {}'.format(change_set_name, stack_name), self._client_type) spinner.start() self._change_set_create.wait(StackName=stack_name, ChangeSetName=change_set_name) spinner.stop() Oprint.info('Change set {} creation completed'.format(change_set_name), self._client_type) except Exception as e: spinner.stop()
def invoke(self, func_name, **kwargs): """Wrapper for invoke""" try: response = self._client.invoke( FunctionName=self.get_lmdo_format_name(func_name), **kwargs) except Exception as e: Oprint.err(e, 'lambda') return response
def process(self): """Load file into memory""" try: if not self._successor: return self.loading_strategy() else: return self._successor.process_next(self.loading_strategy()) except Exception as e: Oprint.err(e, 'lmdo')
def wait(self, stack_name): try: Oprint.info('Start deleting stack {}'.format(stack_name), self._client_type) spinner.start() self._stack_delete.wait(StackName=stack_name) spinner.stop() Oprint.info('Stack {} delete completed'.format(stack_name), self._client_type) except Exception as e: spinner.stop()
def mkdir(path, mode=0777): """Wrapper for mkdir""" try: os.makedirs(path, mode) except OSError as e: if e.errno != errno.EEXIST: Oprint.err(e, 'lmdo') sys.exit(0) return True return True
def create(self): """Create""" if not self.get_apigateway_name(): Oprint.info('No action for api gateway, skip...', 'apigateway') sys.exit(0) swagger_api = self.create_api_by_swagger() swagger_api = self.create_wsgi_api() if swagger_api: self.create_deployment(swagger_api.get('id'), self._config.get('Stage'), swagger_api.get('name'))
def subscribe(self, topic, protocol, endpoint): """Subscription""" topic_arn = self.get_sns_topic_arn(topic) self._client.subscribe(TopicArn=topic_arn, Protocol=protocol, Endpoint=endpoint) Oprint.info( 'Endpoint {} has subscribed to SNS topic {}'.format( endpoint, topic), self.NAME) return True
def __init__(self, template_path, repo_path=None, params_path=None): if not os.path.isfile(template_path): Oprint.err( 'Template not found by given path {}'.format(templated_path), 'cloudformation') self._template_path = template_path self._params_path = params_path # Default to project root if not given self._repo_path = repo_path or './' self._temp_dir = tempfile.mkdtemp()
def wait(self, bucket_name): try: Oprint.info('Bucket {} delete starts'.format(bucket_name), self._client_type) spinner.start() self._bucket_not_exist.wait(Bucket=bucket_name) spinner.stop() Oprint.info('Bucket {} delete completed'.format(bucket_name), self._client_type) except Exception as e: spinner.stop()
def delete_role_and_associated_policies(self, role_name): """Delet a role and associated policies""" try: self.detach_role_managed_policies(role_name) self.delete_role_inline_policies(role_name) response = self.delete_role(role_name) except Exception as e: Oprint.err(e, 'iam', exit=False) return response
def upsert_rule(self, **kwargs): """Create or update rule""" try: name = kwargs.get('Name') Oprint.info('Creating Cloudwatch Event rule {}'.format(name), self.NAME) response = self._client.put_rule(**kwargs) except Exception as e: Oprint.err(e, self.NAME) return response['RuleArn']