def print_new_events(all_events, existing_events):
new_events = [evnt for evnt in all_events if evnt not in existing_events]
for event in new_events:
update = "%s: Resource: %s\t\tStatus: %s" % (
event['Timestamp'], event['LogicalResourceId'],
event['ResourceStatus'])
if 'ResourceStatusReason' in event:
update += "\t\tReason: %s" % event['ResourceStatusReason']
if "ERROR" in update or "FAIL" in update:
log_intent_err(update)
else:
log_intent(update)
def _find_commit_sha(self, version=None):
log_intent("Finding commit SHA")
try:
version_to_find = version or "HEAD"
commit_sha = subprocess.check_output(
["git", "rev-list", "-n", "1",
version_to_find]).strip().decode("utf-8")
log_intent("Found commit SHA " + commit_sha)
return commit_sha
except:
log_err("Commit SHA not found. Given version is not a git tag, \
branch or commit SHA")
exit(1)
def _add_image_tag(self, existing_tag, new_tag):
try:
image_manifest = self.client.batch_get_image(
repositoryName=self.repo_name,
imageIds=[{
'imageTag': existing_tag
}])['images'][0]['imageManifest']
self.client.put_image(repositoryName=self.repo_name,
imageTag=new_tag,
imageManifest=image_manifest)
log_intent(f'Added additional tag: {new_tag}')
except:
log_err("Unable to add additional tag " + str(new_tag))
def ensure_repository(self):
try:
self.ecr_client.create_repository(
repositoryName=self.repo_name,
imageScanningConfiguration={
'scanOnPush': True
},
)
log_intent('Repo created with name: '+self.repo_name)
except Exception as ex:
if type(ex).__name__ == 'RepositoryAlreadyExistsException':
log_intent('Repo exists with name: '+self.repo_name)
else:
raise ex
def run(self):
log_warning("Deploying to {self.region}".format(**locals()))
self.init_stack_info()
if not os.path.exists(self.env_sample_file):
raise UnrecoverableException('env.sample not found. Exiting.')
log_intent("name: " + self.name + " | environment: " +
self.environment + " | version: " + str(self.version))
log_bold("Checking image in ECR")
self.upload_artefacts()
log_bold("Initiating deployment\n")
ecs_client = EcsClient(None, None, self.region)
jobs = []
for index, service_name in enumerate(self.ecs_service_names):
log_bold("Starting to deploy " + service_name)
color = DEPLOYMENT_COLORS[index % 3]
image_url = self.ecr_image_uri
image_url += (':' + self.version)
process = multiprocessing.Process(
target=deployer.deploy_new_version,
args=(
ecs_client,
self.cluster_name,
service_name,
self.version,
self.name,
self.env_sample_file,
self.environment,
color,
image_url
)
)
jobs.append(process)
process.start()
exit_codes = []
while True:
sleep(1)
exit_codes = [proc.exitcode for proc in jobs]
if None not in exit_codes:
break
if any(exit_codes) != 0:
raise UnrecoverableException("Deployment failed")
def _derive_version(self, git_version=None):
log_intent("Finding commit SHA")
try:
version_to_find = git_version or "HEAD"
commit_sha = subprocess.check_output(
["git", "rev-list", "-n", "1",
version_to_find]).strip().decode("utf-8")
derived_version = commit_sha
if self.dockerfile is not None and self.dockerfile != DEFAULT_DOCKER_FILE:
derived_version = "{}-{}".format(derived_version,
self.dockerfile)
log_intent("Derived version is " + derived_version)
return derived_version
except:
raise UnrecoverableException(
"Commit SHA not found. Given version is not a git tag, \
branch or commit SHA")
def update(self):
log_warning(
"Update task definition to {self.region}".format(**locals()))
if not os.path.exists(self.env_sample_file):
raise UnrecoverableException('env.sample not found. Exiting.')
ecr_client = EcrClient(self.name, self.region, self.build_args)
ecr_client.set_version(self.version)
log_intent("name: " + self.name + " | environment: " +
self.environment + " | version: " + str(ecr_client.version))
log_bold("Checking image in ECR")
ecr_client.build_and_upload_image()
log_bold("Updating task definition\n")
env_config = build_config(self.environment, self.name,
self.env_sample_file)
ecs_client = EcsClient(region=self.region)
deployment = DeployAction(ecs_client, self.cluster_name, None)
task_defn = self._apply_changes_over_current_task_defn(
env_config, ecs_client, ecr_client, deployment)
deployment.update_task_definition(task_defn)
log_bold("Task definition successfully updated\n")
def run(self):
log_warning("Deploying to {self.region}".format(**locals()))
if not os.path.exists(self.env_sample_file):
raise UnrecoverableException('env.sample not found. Exiting.')
log_intent("name: " + self.name + " | environment: " +
self.environment + " | version: " + str(self.version) +
" | deployment_identifier: " + self.deployment_identifier)
log_bold("Checking image in ECR")
self.ecr.upload_artefacts()
log_bold("Initiating deployment\n")
ecs_client = EcsClient(None, None, self.region)
image_url = self.ecr.image_uri
target = deployer.deploy_new_version
kwargs = dict(client=ecs_client, cluster_name=self.cluster_name,
service_name=self.name, sample_env_file_path=self.env_sample_file,
timeout_seconds=self.timeout_seconds, env_name=self.environment,
ecr_image_uri=image_url,
deployment_identifier=self.deployment_identifier,
)
self.run_job_for_all_services("Deploy", target, kwargs)
def build_config(env_name, service_name, sample_env_file_path):
service_config = read_config(open(sample_env_file_path).read())
try:
environment_config = ParameterStore(service_name,
env_name).get_existing_config()
except Exception as err:
log_intent(str(err))
raise UnrecoverableException(
"Cannot find the configuration in parameter store \
[env: %s | service: %s]." % (env_name, service_name))
missing_env_config = set(service_config) - set(environment_config)
if missing_env_config:
raise UnrecoverableException('There is no config value for the keys ' +
str(missing_env_config))
missing_env_sample_config = set(environment_config) - set(service_config)
if missing_env_sample_config:
raise UnrecoverableException(
'There is no config value for the keys in env.sample file ' +
str(missing_env_sample_config))
return make_container_defn_env_conf(service_config, environment_config)
def set_version(self, version):
if version:
try:
commit_sha = self._find_commit_sha(version)
except:
commit_sha = version
log_intent("Using commit hash " + commit_sha + " to find image")
image = self._find_image_in_ecr(commit_sha)
if not image:
log_warning("Please build, tag and upload the image for the \
commit " + commit_sha)
raise UnrecoverableException(
"Image for given version could not be found.")
self.version = version
else:
dirty = subprocess.check_output(["git", "status",
"--short"]).decode("utf-8")
if dirty:
self.version = 'dirty'
log_intent("Version parameter was not provided. Determined \
version to be " + self.version + " based on current status")
else:
self.version = self._find_commit_sha()
log_intent("Version parameter was not provided. Determined \
version to be " + self.version + " based on current status")
def prepare_stack_options_for_template(template_body, environment, stack_name):
options = {}
if len(template_body) <= TEMPLATE_BODY_LIMIT:
options['TemplateBody'] = template_body
else:
s3 = get_resource_for('s3', environment)
bucket_name = get_service_templates_bucket_for_environment(environment)
if not bucket_name:
from cloudlift.exceptions import UnrecoverableException
raise UnrecoverableException(
'Configure "service_templates_bucket" in environment configuration to apply changes'
)
bucket = s3.Bucket(bucket_name)
region = get_region_for_environment(environment)
if bucket not in s3.buckets.all():
bucket.create(ACL='private',
Bucket=bucket_name,
CreateBucketConfiguration={
'LocationConstraint': region,
})
s3.BucketVersioning(bucket_name).enable()
with NamedTemporaryFile() as f:
f.write(template_body.encode())
path = '{}/{}.template'.format(environment, stack_name)
bucket.upload_file(f.name, path)
template_url = 'https://s3-{}.amazonaws.com/{}/{}'.format(
region, bucket_name, path)
log_intent(
'Using S3 URL from deploying stack: {}'.format(template_url))
options['TemplateURL'] = template_url
return options
def _ensure_image_in_ecr(self):
if self.version == 'dirty':
image = None
else:
image = self._find_image_in_ecr(self.version)
if image:
log_intent("Image found in ECR")
else:
log_bold(
f"Image not found in ECR. Building image {self.name} {self.version}"
)
image_name = spinalcase(self.name) + ':' + self.version
ecr_name = self.ecr_image_uri + ':' + self.version
self._build_image(image_name)
self._push_image(image_name, ecr_name)
image = self._find_image_in_ecr(self.version)
try:
image_manifest = image['imageManifest']
self.ecr_client.put_image(repositoryName=self.repo_name,
imageTag=self.version,
imageManifest=image_manifest)
except Exception:
pass
def ensure_image_in_ecr(self):
if self.version:
try:
commit_sha = self._find_commit_sha(self.version)
except:
commit_sha = self.version
log_intent("Using commit hash " + commit_sha + " to find image")
image = self._find_image_in_ecr(commit_sha)
if not image:
log_warning("Please build, tag and upload the image for the \
commit " + commit_sha)
raise UnrecoverableException("Image for given version could not be found.")
else:
dirty = subprocess.check_output(
["git", "status", "--short"]
).decode("utf-8")
if dirty:
self.version = 'dirty'
log_intent("Version parameter was not provided. Determined \
version to be " + self.version + " based on current status")
image = None
else:
self.version = self._find_commit_sha()
log_intent("Version parameter was not provided. Determined \
version to be " + self.version + " based on current status")
image = self._find_image_in_ecr(self.version)
if image:
log_intent("Image found in ECR")
else:
log_bold("Image not found in ECR. Building image")
image_name = spinalcase(self.name) + ':' + self.version
ecr_name = self.ecr_image_uri + ':' + self.version
self._build_image(image_name)
self._push_image(image_name, ecr_name)
image = self._find_image_in_ecr(self.version)
try:
image_manifest = image['imageManifest']
self.ecr_client.put_image(
repositoryName=self.repo_name,
imageTag=self.version,
imageManifest=image_manifest
)
except Exception:
pass
def ensure_image_in_ecr(self):
if self.version:
log_intent("Using commit hash " + self.version + " to find image")
image = self._find_image_in_ecr(self.version)
if not image:
log_warning("Please build, tag and upload the image for the \
commit " + self.version)
raise UnrecoverableException(
"Image for given version could not be found.")
else:
dirty = subprocess.check_output(["git", "status",
"--short"]).decode("utf-8")
if dirty:
log_intent(
"Repository has uncommitted changes. Marking version as dirty."
)
self.version = '{}-dirty'.format(self._derive_version())
image = None
else:
self.version = self._derive_version()
image = self._find_image_in_ecr(self.version)
log_intent(
"Version parameter was not provided. Determined version to be "
+ self.version + " based on current status")
if image:
log_intent("Image found in ECR")
else:
log_bold("Image not found in ECR. Building image")
self._build_image()
self._push_image()
image = self._find_image_in_ecr(self.version)
try:
image_manifest = image['imageManifest']
self.client.put_image(repositoryName=self.repo_name,
imageTag=self.version,
imageManifest=image_manifest)
except Exception:
pass
self._add_image_tag(self.version,
f'{self.version}-{self._git_epoch_time()}')
def log_ips(self):
for service in self.ecs_service_names:
task_arns = self.ecs_client.list_tasks(
cluster=self.cluster_name, serviceName=service)['taskArns']
tasks = self.ecs_client.describe_tasks(cluster=self.cluster_name,
tasks=task_arns)['tasks']
container_instance_arns = [
task['containerInstanceArn'] for task in tasks
]
container_instances = self.ecs_client.describe_container_instances(
cluster=self.cluster_name,
containerInstances=container_instance_arns
)['containerInstances']
ecs_instance_ids = [
container['ec2InstanceId'] for container in container_instances
]
ec2_reservations = self.ec2_client.describe_instances(
InstanceIds=ecs_instance_ids)['Reservations']
log_bold(service, )
for reservation in ec2_reservations:
instances = reservation['Instances']
ips = [instance['PrivateIpAddress'] for instance in instances]
[log_intent(ip) for ip in ips]
log("")
def ensure_repository(self):
try:
self.client.create_repository(
repositoryName=self.repo_name,
imageScanningConfiguration={'scanOnPush': True},
)
log_intent('Repo created with name: ' + self.repo_name)
except Exception as ex:
if type(ex).__name__ == 'RepositoryAlreadyExistsException':
log_intent('Repo exists with name: ' + self.repo_name)
else:
raise ex
current_account_id = get_account_id()
if current_account_id != self.account_id:
log_intent('Setting cross account ECR access: ' + self.repo_name)
self.client.set_repository_policy(
repositoryName=self.repo_name,
policyText=json.dumps({
"Version":
"2008-10-17",
"Statement": [{
"Sid":
"AllowCrossAccountPull-{}".format(current_account_id),
"Effect":
"Allow",
"Principal": {
"AWS": [current_account_id]
},
"Action": [
"ecr:GetDownloadUrlForLayer",
"ecr:BatchCheckLayerAvailability",
"ecr:BatchGetImage",
"ecr:InitiateLayerUpload",
"ecr:PutImage",
"ecr:UploadLayerPart",
"ecr:CompleteLayerUpload",
]
}]
}))
def get_current_image_uri(self):
ecr_image_uri = self._fetch_current_image_uri()
log_intent(f"Currently deployed tag: {ecr_image_uri}")
return str(ecr_image_uri)
