def run(self):
try:
log("Check if stack already exists for " + self.cluster_name)
environment_stack = self.client.describe_stacks(
StackName=self.cluster_name)['Stacks'][0]
log(self.cluster_name + " stack exists. ID: " +
environment_stack['StackId'])
log_err("Cannot create environment with duplicate name: " +
self.cluster_name)
except Exception:
log(self.cluster_name +
" stack does not exist. Creating new stack.")
# When creating a cluster, desired_instance count is same
# as min_instance count
environment_stack_template_body = ClusterTemplateGenerator(
self.environment, self.configuration).generate_cluster()
self.existing_events = get_stack_events(self.client,
self.cluster_name)
environment_stack = self.client.create_stack(
StackName=self.cluster_name,
TemplateBody=environment_stack_template_body,
Parameters=[{
'ParameterKey': 'KeyPair',
'ParameterValue': self.key_name,
}, {
'ParameterKey': 'Environment',
'ParameterValue': self.environment,
}],
OnFailure='DO_NOTHING',
Capabilities=['CAPABILITY_NAMED_IAM'],
)
log_bold("Submitted to cloudformation. Checking progress...")
self.__print_progress()
log_bold(self.cluster_name + " stack created. ID: " +
environment_stack['StackId'])
def create(self):
'''
Create and execute CloudFormation template for ECS service
and related dependencies
'''
log_bold("Initiating service creation")
self.service_configuration.edit_config()
template_generator = ServiceTemplateGenerator(
self.service_configuration, self.environment_stack)
service_template_body = template_generator.generate_service()
try:
self.client.create_stack(
StackName=self.stack_name,
TemplateBody=service_template_body,
Parameters=[{
'ParameterKey': 'Environment',
'ParameterValue': self.environment,
}],
OnFailure='DO_NOTHING',
Capabilities=['CAPABILITY_NAMED_IAM'],
)
log_bold("Submitted to cloudformation. Checking progress...")
self._print_progress()
except ClientError as boto_client_error:
error_code = boto_client_error.response['Error']['Code']
if error_code == 'AlreadyExistsException':
log_err("Stack " + self.stack_name + " already exists.")
exit(1)
else:
raise boto_client_error
def update(self):
'''
Create and execute changeset for existing CloudFormation template
for ECS service and related dependencies
'''
log_bold("Starting to update service")
self.service_configuration.edit_config()
try:
template_generator = ServiceTemplateGenerator(
self.service_configuration, self.environment_stack)
service_template_body = template_generator.generate_service()
change_set = create_change_set(self.client, service_template_body,
self.stack_name, "",
self.environment)
self.service_configuration.update_cloudlift_version()
log_bold("Executing changeset. Checking progress...")
self.client.execute_change_set(
ChangeSetName=change_set['ChangeSetId'])
self._print_progress()
except ClientError as exc:
if "No updates are to be performed." in str(exc):
log_err("No updates are to be performed")
else:
raise exc
def run_update(self, update_ecs_agents):
if update_ecs_agents:
self.__run_ecs_container_agent_udpate()
try:
log("Initiating environment stack update.")
environment_stack_template_body = ClusterTemplateGenerator(
self.environment,
self.configuration,
self.__get_desired_count()
).generate_cluster()
log("Template generation complete.")
change_set = create_change_set(
self.client,
environment_stack_template_body,
self.cluster_name,
self.__get_parameter_values(),
self.environment
)
self.existing_events = get_stack_events(
self.client,
self.cluster_name
)
if change_set is None:
return
log_bold("Executing changeset. Checking progress...")
self.client.execute_change_set(
ChangeSetName=change_set['ChangeSetId']
)
self.__print_progress()
except ClientError as e:
log_err("No updates are to be performed")
except Exception as e:
raise e
def deploy_new_version(client, cluster_name, ecs_service_name,
deploy_version_tag, service_name, sample_env_file_path,
env_name, color='white', complete_image_uri=None):
env_config = build_config(env_name, service_name, sample_env_file_path)
deployment = DeployAction(client, cluster_name, ecs_service_name)
if deployment.service.desired_count == 0:
desired_count = 1
else:
desired_count = deployment.service.desired_count
deployment.service.set_desired_count(desired_count)
task_definition = deployment.get_current_task_definition(
deployment.service
)
if complete_image_uri is not None:
container_name = task_definition['containerDefinitions'][0]['name']
task_definition.set_images(
deploy_version_tag,
**{container_name: complete_image_uri}
)
else:
task_definition.set_images(deploy_version_tag)
for container in task_definition.containers:
task_definition.apply_container_environment(container, env_config)
print_task_diff(ecs_service_name, task_definition.diff, color)
new_task_definition = deployment.update_task_definition(task_definition)
response = deploy_and_wait(deployment, new_task_definition, color)
if response:
log_bold(ecs_service_name + " Deployed successfully.")
else:
log_err(ecs_service_name + " Deployment failed.")
return response
def _edit_config(self):
'''
Open editor to update configuration
'''
try:
current_configuration = self.get_config()
updated_configuration = edit(
json.dumps(current_configuration,
indent=4,
sort_keys=True,
cls=DecimalEncoder))
if updated_configuration is None:
log_warning("No changes made.")
else:
updated_configuration = json.loads(updated_configuration)
differences = list(
dictdiffer.diff(current_configuration,
updated_configuration))
if not differences:
log_warning("No changes made.")
else:
print_json_changes(differences)
if confirm('Do you want update the config?'):
self._set_config(updated_configuration)
else:
log_warning("Changes aborted.")
except ClientError:
log_err("Unable to fetch environment configuration from DynamoDB.")
exit(1)
def get_config(self):
'''
Get configuration from DynamoDB
'''
try:
configuration_response = self.table.get_item(
Key={
'service_name': self.service_name,
'environment': self.environment
},
ConsistentRead=True,
AttributesToGet=['configuration'])
if 'Item' in configuration_response:
existing_configuration = configuration_response['Item'][
'configuration']
else:
existing_configuration = self._default_service_configuration()
self.new_service = True
existing_configuration.pop("cloudlift_version", None)
return existing_configuration
except ClientError:
log_err("Unable to fetch service configuration from DynamoDB.")
exit(1)
def _initiate_session(self, target_instance):
log_bold("Starting session in " + target_instance)
try:
driver = create_clidriver()
driver.main(["ssm", "start-session", "--target", target_instance])
except:
log_err("Failed to start session")
exit(1)
def get_ssl_certification_for_environment(environment):
try:
return EnvironmentConfiguration(environment).get_config(
)[environment]['environment']["ssl_certificate_arn"]
except KeyError:
log_err("Unable to find ssl certificate for {environment}".format(
**locals()))
exit(1)
def _get_target_instance(self):
service_instance_ids = ServiceInformationFetcher(
self.name, self.environment).get_instance_ids()
if not service_instance_ids:
log_err("Couldn't find instances. Exiting.")
exit(1)
instance_ids = list(
set(functools.reduce(operator.add, service_instance_ids.values())))
log("Found " + str(len(instance_ids)) + " instances to start session")
return instance_ids[0]
def _push_image(self, local_name, ecr_name):
try:
subprocess.check_call(["docker", "tag", local_name, ecr_name])
except:
log_err("Local image was not found.")
exit(1)
self._login_to_ecr()
subprocess.check_call(["docker", "push", ecr_name])
subprocess.check_call(["docker", "rmi", ecr_name])
log_intent('Pushed the image (' + local_name + ') to ECR sucessfully.')
def wait_for_finish(action, existing_events, color, deploy_end_time):
while time() <= deploy_end_time:
service = action.get_service()
existing_events = fetch_and_print_new_events(service, existing_events,
color)
if is_deployed(service):
return True
sleep(5)
log_err("Deployment timed out!")
return False
def wait_for_finish(action, existing_events, color):
waiting = True
while waiting:
sleep(1)
service = action.get_service()
existing_events = fetch_and_print_new_events(service, existing_events,
color)
waiting = not action.is_deployed(service) and not service.errors
if service.errors:
log_err(str(service.errors))
return False
return True
def cli():
"""
Cloudlift is built by Simpl developers to make it easier to launch \
dockerized services in AWS ECS.
"""
try:
boto3.client('cloudformation')
except ClientError:
log_err("Could not connect to AWS!")
log_err("Ensure AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY & \
AWS_DEFAULT_REGION env vars are set OR run 'aws configure'")
exit(1)
def _add_image_tag(self, existing_tag, new_tag):
try:
image_manifest = self.ecr_client.batch_get_image(
repositoryName=self.repo_name,
imageIds=[{
'imageTag': existing_tag
}])['images'][0]['imageManifest']
self.ecr_client.put_image(repositoryName=self.repo_name,
imageTag=new_tag,
imageManifest=image_manifest)
except:
log_err("Unable to add additional tag " + str(new_tag))
def _find_commit_sha(self, version=None):
log_intent("Finding commit SHA")
try:
version_to_find = version or "HEAD"
commit_sha = subprocess.check_output(
["git", "rev-list", "-n", "1",
version_to_find]).strip().decode("utf-8")
log_intent("Found commit SHA " + commit_sha)
return commit_sha
except:
log_err("Commit SHA not found. Given version is not a git tag, \
branch or commit SHA")
exit(1)
def _get_environment_stack(self):
try:
log("Looking for " + self.environment + " cluster.")
environment_stack = self.client.describe_stacks(
StackName=get_cluster_name(self.environment))['Stacks'][0]
log_bold(self.environment + " stack found. Using stack with ID: " +
environment_stack['StackId'])
except ClientError:
log_err(self.environment +
" cluster not found. Create the environment \
cluster using `create_environment` command.")
exit(1)
return environment_stack
def _print_progress(self):
while True:
response = self.client.describe_stacks(StackName=self.stack_name)
if "IN_PROGRESS" not in response['Stacks'][0]['StackStatus']:
break
all_events = get_stack_events(self.client, self.stack_name)
print_new_events(all_events, self.existing_events)
self.existing_events = all_events
sleep(5)
final_status = response['Stacks'][0]['StackStatus']
if "FAIL" in final_status:
log_err("Finished with status: %s" % (final_status))
else:
log_bold("Finished with status: %s" % (final_status))
def create_change_set(client, service_template_body, stack_name, key_name,
environment):
change_set_parameters = [{
'ParameterKey': 'Environment',
'ParameterValue': environment
}]
if key_name:
change_set_parameters.append({
'ParameterKey': 'KeyPair',
'ParameterValue': key_name
})
create_change_set_res = client.create_change_set(
StackName=stack_name,
ChangeSetName="cg" + uuid.uuid4().hex,
TemplateBody=service_template_body,
Parameters=change_set_parameters,
Capabilities=['CAPABILITY_NAMED_IAM'],
ChangeSetType='UPDATE')
log("Changeset creation initiated. Checking the progress...")
change_set = client.describe_change_set(
ChangeSetName=create_change_set_res['Id'])
while change_set['Status'] in ['CREATE_PENDING', 'CREATE_IN_PROGRESS']:
sleep(1)
status_string = '\x1b[2K\rChecking changeset status. Status: ' + \
change_set['Status']
sys.stdout.write(status_string)
sys.stdout.flush()
change_set = client.describe_change_set(
ChangeSetName=create_change_set_res['Id'])
status_string = '\x1b[2K\rChecking changeset status.. Status: ' + \
change_set['Status']+'\n'
sys.stdout.write(status_string)
if change_set['Status'] == 'FAILED':
log_err("Changeset creation failed!")
log_bold(
change_set.get('StatusReason', "Check AWS console for reason."))
client.delete_change_set(ChangeSetName=create_change_set_res['Id'])
exit(0)
else:
log_bold("Changeset created.. Following are the changes")
_print_changes(change_set)
if click.confirm('Do you want to execute the changeset?'):
return change_set
log_bold("Deleting changeset...")
client.delete_change_set(ChangeSetName=create_change_set_res['Id'])
log_bold("Done. Bye!")
exit(0)
def _set_config(self, config):
'''
Set configuration in DynamoDB
'''
self._validate_changes(config)
try:
configuration_response = self.table.update_item(
TableName=ENVIRONMENT_CONFIGURATION_TABLE,
Key={'environment': self.environment},
UpdateExpression='SET configuration = :configuration',
ExpressionAttributeValues={':configuration': config},
ReturnValues="UPDATED_NEW")
return configuration_response
except ClientError:
log_err("Unable to store environment configuration in DynamoDB.")
exit(1)
pass
def get_mfa_session(mfa_code=None, region='ap-south-1'):
username = get_username()
if not mfa_code:
mfa_code = input("MFA Code: ")
mfa_arn = "arn:aws:iam::%s:mfa/%s" % (get_account_id(), username)
log_bold("Using credentials for " + username)
try:
session_params = client('sts').get_session_token(
DurationSeconds=900, SerialNumber=mfa_arn, TokenCode=str(mfa_code))
credentials = session_params['Credentials']
return Session(aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
region_name=region)
except botocore.exceptions.ClientError as client_error:
log_err(str(client_error))
exit(1)
def _validate_changes(self, differences):
errors = []
for parameter_change in differences:
if parameter_change[0] == 'change':
if not self._is_a_valid_parameter_key(parameter_change[1]):
errors.append("'%s' is not a valid key." % parameter_change[1])
elif parameter_change[0] == 'add':
for added_parameter in parameter_change[2]:
if not self._is_a_valid_parameter_key(added_parameter[0]):
errors.append("'%s' is not a valid key." % added_parameter[0])
elif parameter_change[0] == 'remove':
# No validation required
pass
if errors:
for error in errors:
log_err(error)
raise UnrecoverableException("Environment variables validation failed with above errors.")
return True
def ensure_image_in_ecr(self):
if self.version:
try:
commit_sha = self._find_commit_sha(self.version)
except:
commit_sha = self.version
log_intent("Using commit hash " + commit_sha + " to find image")
image = self._find_image_in_ecr(commit_sha)
if not image:
log_err("Image for given version could not be found.")
log_warning("Please build, tag and upload the image for the \
commit " + commit_sha)
exit(1)
else:
dirty = subprocess.check_output(["git", "status",
"--short"]).decode("utf-8")
if dirty:
self.version = 'dirty'
log_intent("Version parameter was not provided. Determined \
version to be " + self.version + " based on current status")
image = None
else:
self.version = self._find_commit_sha()
log_intent("Version parameter was not provided. Determined \
version to be " + self.version + " based on current status")
image = self._find_image_in_ecr(self.version)
if image:
log_intent("Image found in ECR")
else:
log_bold("Image not found in ECR. Building image")
image_name = spinalcase(self.name) + ':' + self.version
ecr_name = self.ecr_image_uri + ':' + self.version
self._build_image(image_name)
self._push_image(image_name, ecr_name)
image = self._find_image_in_ecr(self.version)
try:
image_manifest = image['imageManifest']
self.ecr_client.put_image(repositoryName=self.repo_name,
imageTag=self.version,
imageManifest=image_manifest)
except Exception:
pass
def do_mfa_login(mfa_code=None, region='ap-south-1'):
username = get_username()
if not mfa_code:
mfa_code = input("MFA Code: ")
mfa_arn = "arn:aws:iam::%s:mfa/%s" % (get_account_id(), username)
log_bold("Using credentials for " + username)
try:
session_params = client('sts').get_session_token(
DurationSeconds=900, SerialNumber=mfa_arn, TokenCode=str(mfa_code))
credentials = session_params['Credentials']
os.environ['AWS_ACCESS_KEY_ID'] = credentials['AccessKeyId']
os.environ['AWS_SECRET_ACCESS_KEY'] = credentials['SecretAccessKey']
os.environ['AWS_SESSION_TOKEN'] = credentials['SessionToken']
os.environ['AWS_DEFAULT_REGION'] = region
return session_params
except botocore.exceptions.ClientError as client_error:
log_err(str(client_error))
exit(1)
def get_config(self):
'''
Get configuration from DynamoDB
'''
try:
configuration_response = self.table.get_item(
Key={'environment': self.environment},
ConsistentRead=True,
AttributesToGet=['configuration'])
return configuration_response['Item']['configuration']
except ClientError:
log_err("Unable to fetch environment configuration from DynamoDB.")
exit(1)
except KeyError:
log_err(
"Environment configuration not found. Does this environment exist?"
)
exit(1)
def update(self):
'''
Create and execute changeset for existing CloudFormation template
for ECS service and related dependencies
'''
log_bold("Starting to update service")
self.service_configuration.edit_config()
self.service_configuration.validate()
information_fetcher = ServiceInformationFetcher(
self.service_configuration.service_name,
self.environment,
self.service_configuration.get_config(),
)
try:
current_image_uri = information_fetcher.get_current_image_uri()
desired_counts = information_fetcher.fetch_current_desired_count()
template_generator = ServiceTemplateGenerator(
self.service_configuration,
self.environment_stack,
self.env_sample_file,
current_image_uri,
desired_counts,
)
service_template_body = template_generator.generate_service()
change_set = create_change_set(self.client, service_template_body,
self.stack_name, None,
self.environment)
if change_set is None:
return
self.service_configuration.update_cloudlift_version()
log_bold("Executing changeset. Checking progress...")
self.client.execute_change_set(
ChangeSetName=change_set['ChangeSetId'])
self._print_progress()
except ClientError as exc:
if "No updates are to be performed." in str(exc):
log_err("No updates are to be performed")
else:
raise exc
def set_config(self, config):
'''
Set configuration in DynamoDB
'''
config['cloudlift_version'] = VERSION
self._validate_changes(config)
try:
configuration_response = self.table.update_item(
TableName=SERVICE_CONFIGURATION_TABLE,
Key={
'service_name': self.service_name,
'environment': self.environment
},
UpdateExpression='SET configuration = :configuration',
ExpressionAttributeValues={':configuration': config},
ReturnValues="UPDATED_NEW")
return configuration_response
except ClientError:
log_err("Unable to store service configuration in DynamoDB.")
exit(1)
def init_stack_info(self):
try:
self.stack_name = get_service_stack_name(self.environment,
self.name)
stack = get_client_for('cloudformation',
self.environment).describe_stacks(
StackName=self.stack_name)['Stacks'][0]
self.ecs_service_names = [
service_name['OutputValue'] for service_name in list(
filter(lambda x: x['OutputKey'].endswith('EcsServiceName'),
stack['Outputs']))
]
except ClientError as client_error:
err = str(client_error)
if "Stack with id %s does not exist" % self.stack_name in err:
log_err(
"%s cluster not found. Create the environment cluster using `create_environment` command."
% self.environment)
else:
raise UnrecoverableException(str(client_error))
def __get_desired_count(self):
try:
auto_scaling_client = get_client_for('autoscaling',
self.environment)
cloudformation_client = get_client_for('cloudformation',
self.environment)
cfn_resources = cloudformation_client.list_stack_resources(
StackName=self.cluster_name)
auto_scaling_group_name = list(
filter(
lambda x: x['ResourceType'
] == "AWS::AutoScaling::AutoScalingGroup",
cfn_resources['StackResourceSummaries'])
)[0]['PhysicalResourceId']
response = auto_scaling_client.describe_auto_scaling_groups(
AutoScalingGroupNames=[auto_scaling_group_name])
return response['AutoScalingGroups'][0]['DesiredCapacity']
except Exception:
log_err("Unable to fetch desired instance count.")
exit(1)
def _validate_changes(self, differences):
errors = []
for parameter_change in differences:
if parameter_change[0] == 'change':
if not self._is_a_valid_parameter_key(parameter_change[1]):
errors.append("'%s' is not a valid key." %
parameter_change[1])
elif parameter_change[0] == 'add':
for added_parameter in parameter_change[2]:
if not self._is_a_valid_parameter_key(added_parameter[0]):
errors.append("'%s' is not a valid key." %
added_parameter[0])
elif parameter_change[0] == 'remove':
# No validation required
pass
if errors:
for error in errors:
log_err(error)
exit(1)
return True