def _fetch_current_image_uri(self):
ecs_client = get_client_for('ecs', self.environment)
if len(self.service_info) < 1:
raise UnrecoverableException(
"cannot get running image_uri: no ECS services found")
ecs_service_name = next(
service_info['ecs_service_name']
for service, service_info in self.service_info.items()
if service_info.get('ecs_service_name'))
ecs_services = ecs_client.describe_services(
cluster=self.cluster_name,
services=[ecs_service_name],
)['services']
if len(ecs_services) < 1:
raise UnrecoverableException(
"cannot get running image_uri: no service found.")
task_definition_arn = ecs_services[0]['taskDefinition']
task_definition = ecs_client.describe_task_definition(
taskDefinition=task_definition_arn)
return task_definition['taskDefinition']['containerDefinitions'][0][
'image']
def _fetch_current_image_uri(self):
ecs_client = get_client_for('ecs', self.environment)
if len(self.service_info) < 1:
raise UnrecoverableException(
"cannot get running image_uri: no ECS services found")
logical_service_name = next(iter(self.service_info))
ecs_service_name = self.service_info[logical_service_name].get(
'ecs_service_name')
task_arns = ecs_client.list_tasks(
cluster=self.cluster_name,
serviceName=ecs_service_name)['taskArns']
if len(task_arns) < 1:
raise UnrecoverableException(
"cannot get running image_uri: no task ARNs found for service")
tasks = ecs_client.describe_tasks(cluster=self.cluster_name,
tasks=task_arns)['tasks']
task_definition_arns = tasks[0]['taskDefinitionArn']
task_definition = ecs_client.describe_task_definition(
taskDefinition=task_definition_arns)
return task_definition['taskDefinition']['containerDefinitions'][0][
'image']
def get_config(self, cloudlift_version=VERSION):
'''
Get configuration from DynamoDB
'''
try:
configuration_response = self.table.get_item(
Key={
'environment': self.environment
},
ConsistentRead=True,
AttributesToGet=[
'configuration'
]
)
existing_configuration = configuration_response['Item']['configuration']
previous_cloudlift_version = existing_configuration.pop("cloudlift_version", None)
# print(f"Previous cloudlift version in environment config is {previous_cloudlift_version}")
if previous_cloudlift_version and LooseVersion(cloudlift_version) < LooseVersion(previous_cloudlift_version):
raise UnrecoverableException(f'Cloudlift Version {previous_cloudlift_version} was used to '
f'create this service. You are using version {cloudlift_version}, '
f'which is older and can cause corruption. Please upgrade to at least '
f'version {previous_cloudlift_version} to proceed.\n\nUpgrade to the '
f'latest version (Recommended):\n'
f'\tpip install -U cloudlift\n\nOR\n\nUpgrade to a compatible version:\n'
f'\tpip install -U cloudlift=={previous_cloudlift_version}')
return existing_configuration
except ClientError:
raise UnrecoverableException("Unable to fetch environment configuration from DynamoDB.")
except KeyError:
raise UnrecoverableException("Environment configuration not found. Does this environment exist?")
def get_client_for(resource, environment):
try:
return boto3.session.Session(
region_name=get_region_for_environment(environment)
).client(resource)
except ClientError as error:
if error.response['Error']['Code'] == 'ExpiredTokenException':
raise UnrecoverableException("AWS session associated with this profile has expired or is otherwise invalid")
elif error.response['Error']['Code'] == 'InvalidIdentityTokenException':
raise UnrecoverableException("AWS token that was passed could not be validated by Amazon Web Services")
elif error.response['Error']['Code'] == 'RegionDisabledException':
raise UnrecoverableException("STS is not activated in the requested region for the account that is being asked to generate credentials")
else:
raise UnrecoverableException("Unable to find valid AWS credentials")
def generate_service(self):
self._add_service_parameters()
self._add_service_outputs()
self._fetch_current_desired_count()
self._add_ecs_service_iam_role()
self._add_cluster_services()
key = uuid.uuid4().hex + '.yml'
if len(to_yaml(self.template.to_json())) > 51000:
try:
self.client.put_object(
Body=to_yaml(self.template.to_json()),
Bucket=self.bucket_name,
Key=key,
)
template_url = f'https://{self.bucket_name}.s3.amazonaws.com/{key}'
return template_url, 'TemplateURL', key
except ClientError as boto_client_error:
error_code = boto_client_error.response['Error']['Code']
if error_code == 'AccessDenied':
raise UnrecoverableException(
f'Unable to store cloudlift service template in S3 bucket at {self.bucket_name}'
)
else:
raise boto_client_error
else:
return to_yaml(self.template.to_json()), 'TemplateBody', ''
def create(self):
'''
Create and execute CloudFormation template for ECS service
and related dependencies
'''
log_bold("Initiating service creation")
self.service_configuration.edit_config()
template_generator = ServiceTemplateGenerator(
self.service_configuration, self.environment_stack)
service_template_body = template_generator.generate_service()
try:
self.client.create_stack(
StackName=self.stack_name,
TemplateBody=service_template_body,
Parameters=[{
'ParameterKey': 'Environment',
'ParameterValue': self.environment,
}],
OnFailure='DO_NOTHING',
Capabilities=['CAPABILITY_NAMED_IAM'],
)
log_bold("Submitted to cloudformation. Checking progress...")
self._print_progress()
except ClientError as boto_client_error:
error_code = boto_client_error.response['Error']['Code']
if error_code == 'AlreadyExistsException':
raise UnrecoverableException("Stack " + self.stack_name +
" already exists.")
else:
raise boto_client_error
def create(self,
config_body=None,
version=None,
build_arg=None,
dockerfile=None,
ssh=None,
cache_from=None):
'''
Create and execute CloudFormation template for ECS service
and related dependencies
'''
log_bold("Initiating service creation")
if config_body is None:
self.service_configuration.edit_config()
else:
self.service_configuration.set_config(config_body)
self.service_configuration.validate()
ecr_repo_config = self.service_configuration.get_config().get(
'ecr_repo')
ecr = ECR(
region=get_region_for_environment(self.environment),
repo_name=ecr_repo_config.get('name'),
account_id=ecr_repo_config.get('account_id', None),
assume_role_arn=ecr_repo_config.get('assume_role_arn', None),
version=version,
build_args=build_arg,
dockerfile=dockerfile,
ssh=ssh,
cache_from=cache_from,
)
ecr.upload_artefacts()
template_generator = ServiceTemplateGenerator(
self.service_configuration, self.environment_stack,
self.env_sample_file, ecr.image_uri)
service_template_body = template_generator.generate_service()
try:
options = prepare_stack_options_for_template(
service_template_body, self.environment, self.stack_name)
self.client.create_stack(
StackName=self.stack_name,
Parameters=[{
'ParameterKey': 'Environment',
'ParameterValue': self.environment,
}],
OnFailure='DO_NOTHING',
Capabilities=['CAPABILITY_NAMED_IAM'],
**options,
)
log_bold("Submitted to cloudformation. Checking progress...")
self._print_progress()
except ClientError as boto_client_error:
error_code = boto_client_error.response['Error']['Code']
if error_code == 'AlreadyExistsException':
raise UnrecoverableException("Stack " + self.stack_name +
" already exists.")
else:
raise boto_client_error
def _initiate_session(self, target_instance):
log_bold("Starting session in " + target_instance)
try:
driver = create_clidriver()
driver.main(["ssm", "start-session", "--target", target_instance])
except:
raise UnrecoverableException("Failed to start session")
def __get_desired_count(self):
try:
auto_scaling_client = get_client_for(
'autoscaling',
self.environment
)
cloudformation_client = get_client_for(
'cloudformation',
self.environment
)
cfn_resources = cloudformation_client.list_stack_resources(
StackName=self.cluster_name
)
auto_scaling_group_name = list(
filter(
lambda x: x['ResourceType'] == "AWS::AutoScaling::AutoScalingGroup",
cfn_resources['StackResourceSummaries']
)
)[0]['PhysicalResourceId']
response = auto_scaling_client.describe_auto_scaling_groups(
AutoScalingGroupNames=[auto_scaling_group_name]
)
return response['AutoScalingGroups'][0]['DesiredCapacity']
except Exception:
raise UnrecoverableException("Unable to fetch desired instance count.")
def get_ssl_certification_for_environment(environment):
try:
return EnvironmentConfiguration(
environment
).get_config()[environment]['environment']["ssl_certificate_arn"]
except KeyError:
raise UnrecoverableException("Unable to find ssl certificate for {environment}".format(**locals()))
def get_environment_level_alb_listener(environment):
env_spec = EnvironmentConfiguration(environment).get_config()[environment]
if 'loadbalancer_listener_arn' not in env_spec:
raise UnrecoverableException('environment level ALB not defined. ' +
'Please run update_environment and set "loadbalancer_listener_arn".')
return env_spec['loadbalancer_listener_arn']
def _edit_config(self):
'''
Open editor to update configuration
'''
try:
current_configuration = self.get_config()
updated_configuration = edit(
json.dumps(
current_configuration,
indent=4,
sort_keys=True,
cls=DecimalEncoder
)
)
if updated_configuration is None:
log_warning("No changes made.")
else:
updated_configuration = json.loads(updated_configuration)
differences = list(dictdiffer.diff(
current_configuration,
updated_configuration
))
if not differences:
log_warning("No changes made.")
else:
print_json_changes(differences)
if confirm('Do you want update the config?'):
self._set_config(updated_configuration)
else:
log_warning("Changes aborted.")
except ClientError:
raise UnrecoverableException("Unable to fetch environment configuration from DynamoDB.")
def get_config(self):
'''
Get configuration from DynamoDB
'''
try:
configuration_response = self.table.get_item(
Key={
'service_name': self.service_name,
'environment': self.environment
},
ConsistentRead=True,
AttributesToGet=['configuration'])
if 'Item' in configuration_response:
existing_configuration = configuration_response['Item'][
'configuration']
else:
existing_configuration = self._default_service_configuration()
self.new_service = True
existing_configuration.pop("cloudlift_version", None)
return existing_configuration
except ClientError:
raise UnrecoverableException(
"Unable to fetch service configuration from DynamoDB.")
def set_version(self, version):
if version:
try:
commit_sha = self._find_commit_sha(version)
except:
commit_sha = version
log_intent("Using commit hash " + commit_sha + " to find image")
image = self._find_image_in_ecr(commit_sha)
if not image:
log_warning("Please build, tag and upload the image for the \
commit " + commit_sha)
raise UnrecoverableException(
"Image for given version could not be found.")
self.version = version
else:
dirty = subprocess.check_output(["git", "status",
"--short"]).decode("utf-8")
if dirty:
self.version = 'dirty'
log_intent("Version parameter was not provided. Determined \
version to be " + self.version + " based on current status")
else:
self.version = self._find_commit_sha()
log_intent("Version parameter was not provided. Determined \
version to be " + self.version + " based on current status")
def __init__(self, name, environment='', env_sample_file='', timeout_seconds=None, version=None,
build_args=None, dockerfile=None, ssh=None, cache_from=None,
deployment_identifier=None, working_dir='.'):
self.name = name
self.environment = environment
self.deployment_identifier = deployment_identifier
self.env_sample_file = env_sample_file
self.timeout_seconds = timeout_seconds
self.version = version
self.ecr_client = boto3.session.Session(region_name=self.region).client('ecr')
self.cluster_name = get_cluster_name(environment)
self.service_configuration = ServiceConfiguration(service_name=name, environment=environment).get_config()
self.service_info_fetcher = ServiceInformationFetcher(self.name, self.environment, self.service_configuration)
if not self.service_info_fetcher.stack_found:
raise UnrecoverableException(
"error finding stack in ServiceUpdater: {}-{}".format(self.name, self.environment))
ecr_repo_config = self.service_configuration.get('ecr_repo')
self.ecr = ECR(
self.region,
ecr_repo_config.get('name', spinalcase(self.name + '-repo')),
ecr_repo_config.get('account_id', get_account_id()),
ecr_repo_config.get('assume_role_arn', None),
version,
build_args,
dockerfile,
working_dir,
ssh,
cache_from
)
def build_secrets_for_all_namespaces(env_name, service_name, ecs_service_name,
sample_env_folder_path, secrets_name):
secrets_across_namespaces = {}
namespaces = get_namespaces_from_directory(sample_env_folder_path)
duplicates = find_duplicate_keys(sample_env_folder_path, namespaces)
if len(duplicates) != 0:
raise UnrecoverableException(
'duplicate keys found in env sample files {} '.format(duplicates))
for namespace in namespaces:
secrets_for_namespace = _get_secrets_for_namespace(
env_name, namespace, sample_env_folder_path, secrets_name)
secrets_across_namespaces.update(secrets_for_namespace)
automated_secret_name = get_automated_injected_secret_name(
env_name, service_name, ecs_service_name)
existing_secrets = {}
try:
existing_secrets = secrets_manager.get_config(automated_secret_name,
env_name)['secrets']
except Exception as err:
log_warning(
f'secret {automated_secret_name} does not exist. It will be created: {err}'
)
if existing_secrets != secrets_across_namespaces:
log(f"Updating {automated_secret_name}")
secrets_manager.set_secrets_manager_config(env_name,
automated_secret_name,
secrets_across_namespaces)
arn = secrets_manager.get_config(automated_secret_name, env_name)['ARN']
return dict(CLOUDLIFT_INJECTED_SECRETS=arn)
def _get_random_available_listener_rule_priority(listener_rules,
listener_arn):
occupied_priorities = set(rule['Priority'] for rule in listener_rules)
available_priorities = set(range(1, 50001)) - occupied_priorities
if not available_priorities:
raise UnrecoverableException(
"No listener rule priorities available for listener_arn: {}".
format(listener_arn))
return int(random.choice(list(available_priorities)))
def _push_image(self, local_name, ecr_name):
try:
subprocess.check_call(["docker", "tag", local_name, ecr_name])
except:
raise UnrecoverableException("Local image was not found.")
self._login_to_ecr()
subprocess.check_call(["docker", "push", ecr_name])
subprocess.check_call(["docker", "rmi", ecr_name])
log_intent('Pushed the image (' + local_name + ') to ECR sucessfully.')
def get_config(self):
'''
Get configuration from DynamoDB
'''
try:
configuration_response = self.table.get_item(
Key={'environment': self.environment},
ConsistentRead=True,
AttributesToGet=['configuration'])
return configuration_response['Item']['configuration']
except ClientError:
raise UnrecoverableException(
"Unable to fetch environment configuration from DynamoDB.")
except KeyError:
raise UnrecoverableException(
"Environment configuration not found. Does this environment exist?"
)
def _get_target_instance(self):
service_instance_ids = ServiceInformationFetcher(
self.name, self.environment).get_instance_ids()
if not service_instance_ids:
raise UnrecoverableException("Couldn't find instances. Exiting.")
instance_ids = list(
set(functools.reduce(operator.add, service_instance_ids.values())))
log("Found " + str(len(instance_ids)) + " instances to start session")
return instance_ids[0]
def _get_parameter_store_config(service_name, env_name):
try:
environment_config, _ = ParameterStore(
service_name, env_name).get_existing_config_paths()
except Exception as err:
log_intent(str(err))
ex_msg = f"Cannot find the configuration in parameter store [env: ${env_name} | service: ${service_name}]."
raise UnrecoverableException(ex_msg)
return environment_config
def run(self):
log_warning("Deploying to {self.region}".format(**locals()))
self.init_stack_info()
if not os.path.exists(self.env_sample_file):
raise UnrecoverableException('env.sample not found. Exiting.')
log_intent("name: " + self.name + " | environment: " +
self.environment + " | version: " + str(self.version))
log_bold("Checking image in ECR")
self.upload_artefacts()
log_bold("Initiating deployment\n")
ecs_client = EcsClient(None, None, self.region)
jobs = []
for index, service_name in enumerate(self.ecs_service_names):
log_bold("Starting to deploy " + service_name)
color = DEPLOYMENT_COLORS[index % 3]
image_url = self.ecr_image_uri
image_url += (':' + self.version)
process = multiprocessing.Process(
target=deployer.deploy_new_version,
args=(
ecs_client,
self.cluster_name,
service_name,
self.version,
self.name,
self.env_sample_file,
self.environment,
color,
image_url
)
)
jobs.append(process)
process.start()
exit_codes = []
while True:
sleep(1)
exit_codes = [proc.exitcode for proc in jobs]
if None not in exit_codes:
break
if any(exit_codes) != 0:
raise UnrecoverableException("Deployment failed")
def verify_env_sample(self, env_sample_directory_path):
if not self.stack_found:
raise UnrecoverableException(
"error finding stack in ServiceUpdater: {}-{}".format(
self.name, self.environment))
service_info = self.service_info
for ecs_service_logical_name in service_info:
ecs_service_info = service_info[ecs_service_logical_name]
secrets_name = ecs_service_info.get('secrets_name')
verify_and_get_secrets_for_all_namespaces(
self.environment, env_sample_directory_path, secrets_name)
def build_config(env_name, service_name, sample_env_file_path):
service_config = read_config(open(sample_env_file_path).read())
try:
environment_config = ParameterStore(service_name,
env_name).get_existing_config()
except Exception as err:
log_intent(str(err))
raise UnrecoverableException(
"Cannot find the configuration in parameter store \
[env: %s | service: %s]." % (env_name, service_name))
missing_env_config = set(service_config) - set(environment_config)
if missing_env_config:
raise UnrecoverableException('There is no config value for the keys ' +
str(missing_env_config))
missing_env_sample_config = set(environment_config) - set(service_config)
if missing_env_sample_config:
raise UnrecoverableException(
'There is no config value for the keys in env.sample file ' +
str(missing_env_sample_config))
return make_container_defn_env_conf(service_config, environment_config)
def _find_commit_sha(self, version=None):
log_intent("Finding commit SHA")
try:
version_to_find = version or "HEAD"
commit_sha = subprocess.check_output(
["git", "rev-list", "-n", "1", version_to_find]
).strip().decode("utf-8")
log_intent("Found commit SHA " + commit_sha)
return commit_sha
except:
raise UnrecoverableException("Commit SHA not found. Given version is not a git tag, \
branch or commit SHA")
def _get_environment_stack(self):
try:
log("Looking for " + self.environment + " cluster.")
environment_stack = self.client.describe_stacks(
StackName=get_cluster_name(self.environment))['Stacks'][0]
log_bold(self.environment + " stack found. Using stack with ID: " +
environment_stack['StackId'])
except ClientError:
raise UnrecoverableException(
self.environment +
" cluster not found. Create the environment \
cluster using `create_environment` command.")
return environment_stack
def get_config(self, cloudlift_version):
'''
Get configuration from DynamoDB
'''
try:
configuration_response = self.table.get_item(
Key={
'service_name': self.service_name,
'environment': self.environment
},
ConsistentRead=True,
AttributesToGet=['configuration'])
if 'Item' in configuration_response:
existing_configuration = configuration_response['Item'][
'configuration']
from distutils.version import LooseVersion
previous_cloudlift_version = existing_configuration.pop(
"cloudlift_version", None)
if LooseVersion(cloudlift_version) < LooseVersion(
previous_cloudlift_version):
raise UnrecoverableException(
f'Cloudlift Version {previous_cloudlift_version} was used to '
f'create this service. You are using version {cloudlift_version}, '
f'which is older and can cause corruption. Please upgrade to at least '
f'version {previous_cloudlift_version} to proceed.\n\nUpgrade to the '
f'latest version (Recommended):\n'
f'\tpip install -U cloudlift\n\nOR\n\nUpgrade to a compatible version:\n'
f'\tpip install -U cloudlift=={previous_cloudlift_version}'
)
else:
existing_configuration = self._default_service_configuration()
self.new_service = True
return existing_configuration
except ClientError:
raise UnrecoverableException(
"Unable to fetch service configuration from DynamoDB.")
def verify_and_get_secrets_for_all_namespaces(env_name, sample_env_folder_path,
secrets_name):
secrets_across_namespaces = {}
namespaces = get_namespaces_from_directory(sample_env_folder_path)
duplicates = find_duplicate_keys(sample_env_folder_path, namespaces)
if len(duplicates) != 0:
raise UnrecoverableException(
'duplicate keys found in env sample files {} '.format(duplicates))
for namespace in namespaces:
secrets_for_namespace = _get_secrets_for_namespace(
env_name, namespace, sample_env_folder_path, secrets_name)
secrets_across_namespaces.update(secrets_for_namespace)
return secrets_across_namespaces
def get_mfa_session(mfa_code=None, region='ap-south-1'):
username = get_username()
if not mfa_code:
mfa_code = input("MFA Code: ")
mfa_arn = "arn:aws:iam::%s:mfa/%s" % (get_account_id(), username)
log_bold("Using credentials for " + username)
try:
session_params = client('sts').get_session_token(
DurationSeconds=900, SerialNumber=mfa_arn, TokenCode=str(mfa_code))
credentials = session_params['Credentials']
return Session(aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
region_name=region)
except botocore.exceptions.ClientError as client_error:
raise UnrecoverableException(str(client_error))
def get_config_in_db(self):
'''
Get configuration from DynamoDB
'''
try:
configuration_response = self.table.get_item(
Key={k: v for k, v in self.kv_pairs},
ConsistentRead=True,
AttributesToGet=[
'configuration'
]
)
if 'Item' in configuration_response:
return configuration_response['Item']['configuration']
return None
except ClientError:
raise UnrecoverableException("Unable to fetch configuration from DynamoDB.")