def upload_file(self, bucket_name, file_path, key, **kwargs):
"""Upload file to S3, provide network progress bar"""
# Check if bucket exist, create one if user agrees
if not self.if_bucket_exist(bucket_name):
sys_pause(
'Bucket {} doesn\'t exist! Do you want to create it? [yes/no]'.
format(bucket_name), 'yes')
self.create_bucket(bucket_name)
file_size = os.path.getsize(file_path) / 1000000
if round(file_size) <= 0:
file_size = 'size:{}B'.format(os.path.getsize(file_path))
else:
file_size = 'size:{}MB'.format(file_size)
Oprint.info(
'Start uploading {} to S3 bucket {}. ({})'.format(
key, bucket_name, file_size), 's3')
#waiter = S3WaiterObjectCreate(self._client)
self._client.upload_file(file_path,
bucket_name,
key,
Callback=FileUploadProgress(file_path),
**kwargs)
#waiter.wait(bucket_name, key)
Oprint.info('Complete uploading {}. ({})'.format(key, file_size), 's3')
return True
def get_stack_output(self, stack_name, key):
"""
Duplicate function to cloudformation due to avoiding circular
condition as cloudformation import this module as well
"""
try:
if not self._stack_info_cache.get(stack_name):
self._stack_info_cache[stack_name] = AWSBase().get_client(
'cloudformation').describe_stacks(StackName=stack_name)
outputs = self._stack_info_cache[stack_name]['Stacks'][0][
'Outputs']
for opts in outputs:
if opts['OutputKey'] == key:
return opts['OutputValue']
Oprint.warn(
'Cannot find key {} output from stack {}'.format(
key, stack_name), 'lmdo')
except Exception:
Oprint.warn(
'Error while retrieving output from {}'.format(stack_name),
'lmdo')
return None
def execute(self):
Oprint.info('Start deploying service', 'lmdo')
self._dispatcher.run(CreateCommand(self._cloudformation))
self._dispatcher.run(CreateCommand(self._lambda))
self._dispatcher.run(CreateCommand(self._apigateway))
self._dispatcher.run(CreateCommand(self._cloudwatchevent))
Oprint.info('Complete deploying service', 'lmdo')
def create_wsgi_api(self):
"""Create/Update api definition for wsgi app"""
swagger_api = False
# If there is already an exsiting swagger api template,
# fetch it so we won't duplicate it
#if os.path.isfile(self.get_swagger_template()) and self.get_apigateway_name():
swagger_api = self.if_api_exist_by_name(self.get_apigateway_name())
iam = IAM()
for lm_func in self._config.get('Lambda'):
if lm_func.get('Type') != AWSLambda.FUNCTION_TYPE_WSGI or lm_func.get('DisableApiGateway'):
continue
function_name = self.get_lmdo_format_name(lm_func.get('FunctionName'))
role = iam.get_lambda_apigateway_default_role(function_name)
Oprint.info('Create/Update API Gateway for wsgi function {}'.format(lm_func.get('FunctionName')), 'apigateway')
to_replace = {
"$title": self.get_apigateway_name(),
"$version": str(datetime.datetime.utcnow()),
"$basePath": lm_func.get('ApiBasePath') or '/res',
"$apiRegion": self.get_region(),
"$functionRegion": self.get_region(),
"$accountId": self.get_account_id(),
"$functionName": function_name,
"$credentials": role['Role'].get('Arn')
}
# Enable cognito user pool as authorizer
if lm_func.get('CognitoUserPoolId'):
se_replace = {
"$apiRegion": self.get_region(),
"$accountId": self.get_account_id(),
"$userPoolId": lm_func.get('CognitoUserPoolId'),
"$CognitoUserPool": 'CognitoUserPool-{}'.format(lm_func.get('FunctionName'))
}
to_replace["$securityDefinitions"] = self.get_apigateway_authorizer(se_replace)
to_replace["$authorizer"] = '{"' + str(se_replace['$CognitoUserPool'])+'":[]}'
else:
to_replace["$securityDefinitions"] = ''
to_replace["$authorizer"] = ''
template_dir = get_template(APIGATEWAY_SWAGGER_WSGI)
if not template_dir:
Oprint.err('Template {} for creating wsgi APIGateway hasn\'t been installed or missing'.format(APIGATEWAY_SWAGGER_WSGI), 'apigateway')
with open(template_dir, 'r') as outfile:
body = update_template(outfile.read(), to_replace)
if not swagger_api:
swagger_api = self.import_rest_api(body)
else:
# Always overwrite for update
self.put_rest_api(swagger_api.get('id'), body, 'merge')
return swagger_api
def create_stack(self, stack_name, capabilities=None, **kwargs):
"""Create stack"""
try:
capabilities = capabilities or ['CAPABILITY_NAMED_IAM', 'CAPABILITY_IAM']
if self._args.get('-he') or self._args.get('--hide-event'):
waiter = CloudformationWaiterStackCreate(self._client)
response = self._client.create_stack(
StackName=stack_name,
Capabilities=capabilities,
**kwargs
)
waiter.wait(stack_name)
else:
response = self._client.create_stack(
StackName=stack_name,
Capabilities=capabilities,
**kwargs
)
self.stack_events_waiter(stack_name=stack_name)
self.lock_stack(stack_name=stack_name)
except Exception as e:
Oprint.err(e, self.NAME)
return False
self.verify_stack(mode='create', stack_id=response.get('StackId'))
return True
def delete_stack(self, stack_name, no_policy=False):
"""Remove a stack by given name"""
# Don't do anything if doesn't exist
stack_info = self.get_stack(stack_name=stack_name)
if not stack_info:
return True
try:
if not no_policy:
self.unlock_stack(stack_name=stack_name)
if self._args.get('-he') or self._args.get('--hide-event'):
waiter = CloudformationWaiterStackDelete(self._client)
response = self._client.delete_stack(StackName=stack_name)
waiter.wait(stack_name)
else:
response = self._client.delete_stack(StackName=stack_name)
self.stack_events_waiter(stack_name=stack_name)
except Exception as e:
Oprint.err(e, self.NAME)
return False
self.verify_stack(mode='delete', stack_id=stack_info['Stacks'][0]['StackId'])
return True
def init(self):
"""Initiating the project and provide a sample lmdo.yaml file"""
if self._args.get('<project_name>'):
mkdir('./{}'.format(self._args.get('<project_name>')))
"""Copy lmdo.yaml over"""
# Do not copy over unless it's a clearn dir
if os.path.isfile(
os.path.join(self._args.get('<project_name>'),
PROJECT_CONFIG_FILE)):
Oprint.err(
'Your have existing {} already, exiting...'.format(
PROJECT_CONFIG_FILE), 'lmdo')
pkg_dir = self.get_installed_path()
if pkg_dir:
copytree(os.path.join(pkg_dir, 'template'),
'./{}'.format(self._args.get('<project_name>')))
elif self._args.get('config'):
pkg_dir = self.get_installed_path()
# Don't override existing lmdo.yaml
if os.path.isfile(PROJECT_CONFIG_FILE):
Oprint.warn(
'You have existing {} file, a copy will be created with name {}.copy'
.format(PROJECT_CONFIG_FILE, PROJECT_CONFIG_FILE), 'lmdo')
shutil.copyfile(
os.path.join(pkg_dir, 'template', PROJECT_CONFIG_FILE),
'{}.copy'.format(PROJECT_CONFIG_FILE))
else:
shutil.copyfile(
os.path.join(pkg_dir, 'template', PROJECT_CONFIG_FILE),
PROJECT_CONFIG_FILE)
def validate_template(self, template_body):
"""Validate template via content"""
try:
result = self._client.validate_template(TemplateBody=template_body)
except Exception as e:
Oprint.err(e, self.NAME)
return True
def validate(self):
"""
Set default value and check if mandatory keys exist
"""
# Set default profile if doesn't exist
if not self.get('Profile'):
self._config['Profile'] = 'default'
# Set default user if doesn't exist
if not self.get('User'):
self._config['User'] = '******'
# Set default service if doesn't exist
if not self.get('Service'):
self._config['Service'] = 'default'
# Set default stage if doesn't exist
if not self.get('Stage'):
self._config['Stage'] = 'dev'
# Check if all keys available
for key in CONFIG_MANDATORY_KEYS:
if key not in self._config:
Oprint.err('{} is missing from config file'.format(key),
'lmdo')
def loading_strategy(self):
"""
Load file into json object
Returns a tuple raw and json
"""
try:
if not self.file_allowed():
raise Exception('File type {} is not allowed'.format(
self.get_ext()))
with open(self._file_path, 'r') as outfile:
raw = outfile.read()
if self.isYaml():
if self._yaml_replacements:
for key, value in self._yaml_replacements.iteritems():
raw = raw.replace(key, value)
json_content = FileLoader.toJson(raw,
file_name=self._file_path)
return raw, json_content
except Exception as e:
Oprint.err(e)
else:
raise Exception('File type {} is not allowed'.format(
self.get_ext()))
def execute(self):
Oprint.info('Start tear down service', 'lmdo')
self._dispatcher.run(DeleteCommand(self._apigateway))
self._dispatcher.run(DeleteCommand(self._lambda))
self._dispatcher.run(DeleteCommand(self._cloudformation))
self._dispatcher.run(DeleteCommand(self._cloudwatchevent))
Oprint.info('Service has been destroy', 'lmdo')
def create_apigateway_lambda_role(self, role_name):
"""Create APIGateway role that can invoke lambda"""
try:
response = self.get_role(role_name)
if response:
Oprint.warn(
'Role {} exists, no action required'.format(role_name),
'iam')
template = get_template(IAM_ROLE_APIGATEWAY_LAMBDA)
if not template:
return False
if not response:
with open(template, 'r') as outfile:
assume_policy = outfile.read()
response = self.create_role(role_name, assume_policy)
policy = self.create_default_policy(
role_name, self.create_policy_name(role_name, 'lambda-invoke'),
IAM_POLICY_APIGATEWAY_LAMBDA_INVOKE)
except Exception as e:
Oprint.err(e, 'iam')
return response
def create_default_events_role(self, role_name):
"""Create lmdo default event role"""
try:
response = self.get_role(role_name)
if response:
Oprint.warn(
'Role {} exists, no action required'.format(role_name),
'iam')
return response
template = get_template(IAM_ROLE_EVENTS)
if not template:
return False
if not response:
with open(template, 'r') as outfile:
assume_policy = outfile.read()
response = self.create_role(role_name, assume_policy)
policy = self.create_default_policy(
role_name, self.create_policy_name(role_name,
'default-events'),
IAM_POLICY_EVENTS)
except Exception as e:
Oprint.err(e, 'apigateway')
return response
def if_api_exist_by_name(self, api_name):
"""
Check if an API exist by given name
Note: APIGateway allows name to be duplicated, it uses internal ID
to identify the actual APIs. We won't consider using same name for
APIs, hence return the first item which should be unique
"""
try:
limit = 100
found = False
pos = None
while True:
if pos:
response = self._client.get_rest_apis(position=pos, limit=limit)
else:
response = self._client.get_rest_apis(limit=limit)
pos = response.get('position')
if len(response.get('items')) > 0:
for api in response.get('items'):
if api_name == api.get('name'):
found = api
break
if not pos:
break
return found
except Exception as e:
Oprint.err(e, 'apigateway')
return False
def describe_change_set(self, change_set_name, *args, **kwargs):
"""Get change set information"""
try:
response = self._client.describe_change_set(ChangeSetName=change_set_name, *args, **kwargs)
except Exception as e:
Oprint.err(e, self.NAME)
return response
def list_existing_change_set(self, stack_name, *args, **kwargs):
"""List all existing stack"""
try:
response = self._client.list_change_sets(StackName=stack_name, *args, **kwargs)
except Exception as e:
Oprint.warn(e, self.NAME)
return response
def create(self):
"""Create/Update stack"""
# Don't run if we don't have templates
if not self._config.get('CloudFormation'):
Oprint.info('No cloudformation found, skip', self.NAME)
return True
self.process()
def delete_change_set(self, change_set_name, *args, **kwargs):
"""Delete existing change set"""
try:
response = self._client.delete_change_set(ChangeSetName=change_set_name, *args, **kwargs)
except Exception as e:
Oprint.warn(e, self.NAME)
return True
def list_functions(self, **kwargs):
"""Wrapper for listing functions"""
try:
response = self._client.list_functions(**kwargs)
except Exception as e:
Oprint.err(e, 'lambda')
return response
def wait(self, change_set_name, stack_name):
try:
Oprint.info('Start creating change set {} for stack {}'.format(change_set_name, stack_name), self._client_type)
spinner.start()
self._change_set_create.wait(StackName=stack_name, ChangeSetName=change_set_name)
spinner.stop()
Oprint.info('Change set {} creation completed'.format(change_set_name), self._client_type)
except Exception as e:
spinner.stop()
def invoke(self, func_name, **kwargs):
"""Wrapper for invoke"""
try:
response = self._client.invoke(
FunctionName=self.get_lmdo_format_name(func_name), **kwargs)
except Exception as e:
Oprint.err(e, 'lambda')
return response
def process(self):
"""Load file into memory"""
try:
if not self._successor:
return self.loading_strategy()
else:
return self._successor.process_next(self.loading_strategy())
except Exception as e:
Oprint.err(e, 'lmdo')
def wait(self, stack_name):
try:
Oprint.info('Start deleting stack {}'.format(stack_name), self._client_type)
spinner.start()
self._stack_delete.wait(StackName=stack_name)
spinner.stop()
Oprint.info('Stack {} delete completed'.format(stack_name), self._client_type)
except Exception as e:
spinner.stop()
def mkdir(path, mode=0777):
"""Wrapper for mkdir"""
try:
os.makedirs(path, mode)
except OSError as e:
if e.errno != errno.EEXIST:
Oprint.err(e, 'lmdo')
sys.exit(0)
return True
return True
def create(self):
"""Create"""
if not self.get_apigateway_name():
Oprint.info('No action for api gateway, skip...', 'apigateway')
sys.exit(0)
swagger_api = self.create_api_by_swagger()
swagger_api = self.create_wsgi_api()
if swagger_api:
self.create_deployment(swagger_api.get('id'), self._config.get('Stage'), swagger_api.get('name'))
def subscribe(self, topic, protocol, endpoint):
"""Subscription"""
topic_arn = self.get_sns_topic_arn(topic)
self._client.subscribe(TopicArn=topic_arn,
Protocol=protocol,
Endpoint=endpoint)
Oprint.info(
'Endpoint {} has subscribed to SNS topic {}'.format(
endpoint, topic), self.NAME)
return True
def __init__(self, template_path, repo_path=None, params_path=None):
if not os.path.isfile(template_path):
Oprint.err(
'Template not found by given path {}'.format(templated_path),
'cloudformation')
self._template_path = template_path
self._params_path = params_path
# Default to project root if not given
self._repo_path = repo_path or './'
self._temp_dir = tempfile.mkdtemp()
def wait(self, bucket_name):
try:
Oprint.info('Bucket {} delete starts'.format(bucket_name),
self._client_type)
spinner.start()
self._bucket_not_exist.wait(Bucket=bucket_name)
spinner.stop()
Oprint.info('Bucket {} delete completed'.format(bucket_name),
self._client_type)
except Exception as e:
spinner.stop()
def delete_role_and_associated_policies(self, role_name):
"""Delet a role and associated policies"""
try:
self.detach_role_managed_policies(role_name)
self.delete_role_inline_policies(role_name)
response = self.delete_role(role_name)
except Exception as e:
Oprint.err(e, 'iam', exit=False)
return response
def upsert_rule(self, **kwargs):
"""Create or update rule"""
try:
name = kwargs.get('Name')
Oprint.info('Creating Cloudwatch Event rule {}'.format(name),
self.NAME)
response = self._client.put_rule(**kwargs)
except Exception as e:
Oprint.err(e, self.NAME)
return response['RuleArn']
