def print_new_events(all_events, existing_events): new_events = [evnt for evnt in all_events if evnt not in existing_events] for event in new_events: update = "%s: Resource: %s\t\tStatus: %s" % ( event['Timestamp'], event['LogicalResourceId'], event['ResourceStatus']) if 'ResourceStatusReason' in event: update += "\t\tReason: %s" % event['ResourceStatusReason'] if "ERROR" in update or "FAIL" in update: log_intent_err(update) else: log_intent(update)
def _find_commit_sha(self, version=None): log_intent("Finding commit SHA") try: version_to_find = version or "HEAD" commit_sha = subprocess.check_output( ["git", "rev-list", "-n", "1", version_to_find]).strip().decode("utf-8") log_intent("Found commit SHA " + commit_sha) return commit_sha except: log_err("Commit SHA not found. Given version is not a git tag, \ branch or commit SHA") exit(1)
def _add_image_tag(self, existing_tag, new_tag): try: image_manifest = self.client.batch_get_image( repositoryName=self.repo_name, imageIds=[{ 'imageTag': existing_tag }])['images'][0]['imageManifest'] self.client.put_image(repositoryName=self.repo_name, imageTag=new_tag, imageManifest=image_manifest) log_intent(f'Added additional tag: {new_tag}') except: log_err("Unable to add additional tag " + str(new_tag))
def ensure_repository(self): try: self.ecr_client.create_repository( repositoryName=self.repo_name, imageScanningConfiguration={ 'scanOnPush': True }, ) log_intent('Repo created with name: '+self.repo_name) except Exception as ex: if type(ex).__name__ == 'RepositoryAlreadyExistsException': log_intent('Repo exists with name: '+self.repo_name) else: raise ex
def run(self): log_warning("Deploying to {self.region}".format(**locals())) self.init_stack_info() if not os.path.exists(self.env_sample_file): raise UnrecoverableException('env.sample not found. Exiting.') log_intent("name: " + self.name + " | environment: " + self.environment + " | version: " + str(self.version)) log_bold("Checking image in ECR") self.upload_artefacts() log_bold("Initiating deployment\n") ecs_client = EcsClient(None, None, self.region) jobs = [] for index, service_name in enumerate(self.ecs_service_names): log_bold("Starting to deploy " + service_name) color = DEPLOYMENT_COLORS[index % 3] image_url = self.ecr_image_uri image_url += (':' + self.version) process = multiprocessing.Process( target=deployer.deploy_new_version, args=( ecs_client, self.cluster_name, service_name, self.version, self.name, self.env_sample_file, self.environment, color, image_url ) ) jobs.append(process) process.start() exit_codes = [] while True: sleep(1) exit_codes = [proc.exitcode for proc in jobs] if None not in exit_codes: break if any(exit_codes) != 0: raise UnrecoverableException("Deployment failed")
def _derive_version(self, git_version=None): log_intent("Finding commit SHA") try: version_to_find = git_version or "HEAD" commit_sha = subprocess.check_output( ["git", "rev-list", "-n", "1", version_to_find]).strip().decode("utf-8") derived_version = commit_sha if self.dockerfile is not None and self.dockerfile != DEFAULT_DOCKER_FILE: derived_version = "{}-{}".format(derived_version, self.dockerfile) log_intent("Derived version is " + derived_version) return derived_version except: raise UnrecoverableException( "Commit SHA not found. Given version is not a git tag, \ branch or commit SHA")
def update(self): log_warning( "Update task definition to {self.region}".format(**locals())) if not os.path.exists(self.env_sample_file): raise UnrecoverableException('env.sample not found. Exiting.') ecr_client = EcrClient(self.name, self.region, self.build_args) ecr_client.set_version(self.version) log_intent("name: " + self.name + " | environment: " + self.environment + " | version: " + str(ecr_client.version)) log_bold("Checking image in ECR") ecr_client.build_and_upload_image() log_bold("Updating task definition\n") env_config = build_config(self.environment, self.name, self.env_sample_file) ecs_client = EcsClient(region=self.region) deployment = DeployAction(ecs_client, self.cluster_name, None) task_defn = self._apply_changes_over_current_task_defn( env_config, ecs_client, ecr_client, deployment) deployment.update_task_definition(task_defn) log_bold("Task definition successfully updated\n")
def run(self): log_warning("Deploying to {self.region}".format(**locals())) if not os.path.exists(self.env_sample_file): raise UnrecoverableException('env.sample not found. Exiting.') log_intent("name: " + self.name + " | environment: " + self.environment + " | version: " + str(self.version) + " | deployment_identifier: " + self.deployment_identifier) log_bold("Checking image in ECR") self.ecr.upload_artefacts() log_bold("Initiating deployment\n") ecs_client = EcsClient(None, None, self.region) image_url = self.ecr.image_uri target = deployer.deploy_new_version kwargs = dict(client=ecs_client, cluster_name=self.cluster_name, service_name=self.name, sample_env_file_path=self.env_sample_file, timeout_seconds=self.timeout_seconds, env_name=self.environment, ecr_image_uri=image_url, deployment_identifier=self.deployment_identifier, ) self.run_job_for_all_services("Deploy", target, kwargs)
def build_config(env_name, service_name, sample_env_file_path): service_config = read_config(open(sample_env_file_path).read()) try: environment_config = ParameterStore(service_name, env_name).get_existing_config() except Exception as err: log_intent(str(err)) raise UnrecoverableException( "Cannot find the configuration in parameter store \ [env: %s | service: %s]." % (env_name, service_name)) missing_env_config = set(service_config) - set(environment_config) if missing_env_config: raise UnrecoverableException('There is no config value for the keys ' + str(missing_env_config)) missing_env_sample_config = set(environment_config) - set(service_config) if missing_env_sample_config: raise UnrecoverableException( 'There is no config value for the keys in env.sample file ' + str(missing_env_sample_config)) return make_container_defn_env_conf(service_config, environment_config)
def set_version(self, version): if version: try: commit_sha = self._find_commit_sha(version) except: commit_sha = version log_intent("Using commit hash " + commit_sha + " to find image") image = self._find_image_in_ecr(commit_sha) if not image: log_warning("Please build, tag and upload the image for the \ commit " + commit_sha) raise UnrecoverableException( "Image for given version could not be found.") self.version = version else: dirty = subprocess.check_output(["git", "status", "--short"]).decode("utf-8") if dirty: self.version = 'dirty' log_intent("Version parameter was not provided. Determined \ version to be " + self.version + " based on current status") else: self.version = self._find_commit_sha() log_intent("Version parameter was not provided. Determined \ version to be " + self.version + " based on current status")
def prepare_stack_options_for_template(template_body, environment, stack_name): options = {} if len(template_body) <= TEMPLATE_BODY_LIMIT: options['TemplateBody'] = template_body else: s3 = get_resource_for('s3', environment) bucket_name = get_service_templates_bucket_for_environment(environment) if not bucket_name: from cloudlift.exceptions import UnrecoverableException raise UnrecoverableException( 'Configure "service_templates_bucket" in environment configuration to apply changes' ) bucket = s3.Bucket(bucket_name) region = get_region_for_environment(environment) if bucket not in s3.buckets.all(): bucket.create(ACL='private', Bucket=bucket_name, CreateBucketConfiguration={ 'LocationConstraint': region, }) s3.BucketVersioning(bucket_name).enable() with NamedTemporaryFile() as f: f.write(template_body.encode()) path = '{}/{}.template'.format(environment, stack_name) bucket.upload_file(f.name, path) template_url = 'https://s3-{}.amazonaws.com/{}/{}'.format( region, bucket_name, path) log_intent( 'Using S3 URL from deploying stack: {}'.format(template_url)) options['TemplateURL'] = template_url return options
def _ensure_image_in_ecr(self): if self.version == 'dirty': image = None else: image = self._find_image_in_ecr(self.version) if image: log_intent("Image found in ECR") else: log_bold( f"Image not found in ECR. Building image {self.name} {self.version}" ) image_name = spinalcase(self.name) + ':' + self.version ecr_name = self.ecr_image_uri + ':' + self.version self._build_image(image_name) self._push_image(image_name, ecr_name) image = self._find_image_in_ecr(self.version) try: image_manifest = image['imageManifest'] self.ecr_client.put_image(repositoryName=self.repo_name, imageTag=self.version, imageManifest=image_manifest) except Exception: pass
def ensure_image_in_ecr(self): if self.version: try: commit_sha = self._find_commit_sha(self.version) except: commit_sha = self.version log_intent("Using commit hash " + commit_sha + " to find image") image = self._find_image_in_ecr(commit_sha) if not image: log_warning("Please build, tag and upload the image for the \ commit " + commit_sha) raise UnrecoverableException("Image for given version could not be found.") else: dirty = subprocess.check_output( ["git", "status", "--short"] ).decode("utf-8") if dirty: self.version = 'dirty' log_intent("Version parameter was not provided. Determined \ version to be " + self.version + " based on current status") image = None else: self.version = self._find_commit_sha() log_intent("Version parameter was not provided. Determined \ version to be " + self.version + " based on current status") image = self._find_image_in_ecr(self.version) if image: log_intent("Image found in ECR") else: log_bold("Image not found in ECR. Building image") image_name = spinalcase(self.name) + ':' + self.version ecr_name = self.ecr_image_uri + ':' + self.version self._build_image(image_name) self._push_image(image_name, ecr_name) image = self._find_image_in_ecr(self.version) try: image_manifest = image['imageManifest'] self.ecr_client.put_image( repositoryName=self.repo_name, imageTag=self.version, imageManifest=image_manifest ) except Exception: pass
def ensure_image_in_ecr(self): if self.version: log_intent("Using commit hash " + self.version + " to find image") image = self._find_image_in_ecr(self.version) if not image: log_warning("Please build, tag and upload the image for the \ commit " + self.version) raise UnrecoverableException( "Image for given version could not be found.") else: dirty = subprocess.check_output(["git", "status", "--short"]).decode("utf-8") if dirty: log_intent( "Repository has uncommitted changes. Marking version as dirty." ) self.version = '{}-dirty'.format(self._derive_version()) image = None else: self.version = self._derive_version() image = self._find_image_in_ecr(self.version) log_intent( "Version parameter was not provided. Determined version to be " + self.version + " based on current status") if image: log_intent("Image found in ECR") else: log_bold("Image not found in ECR. Building image") self._build_image() self._push_image() image = self._find_image_in_ecr(self.version) try: image_manifest = image['imageManifest'] self.client.put_image(repositoryName=self.repo_name, imageTag=self.version, imageManifest=image_manifest) except Exception: pass self._add_image_tag(self.version, f'{self.version}-{self._git_epoch_time()}')
def log_ips(self): for service in self.ecs_service_names: task_arns = self.ecs_client.list_tasks( cluster=self.cluster_name, serviceName=service)['taskArns'] tasks = self.ecs_client.describe_tasks(cluster=self.cluster_name, tasks=task_arns)['tasks'] container_instance_arns = [ task['containerInstanceArn'] for task in tasks ] container_instances = self.ecs_client.describe_container_instances( cluster=self.cluster_name, containerInstances=container_instance_arns )['containerInstances'] ecs_instance_ids = [ container['ec2InstanceId'] for container in container_instances ] ec2_reservations = self.ec2_client.describe_instances( InstanceIds=ecs_instance_ids)['Reservations'] log_bold(service, ) for reservation in ec2_reservations: instances = reservation['Instances'] ips = [instance['PrivateIpAddress'] for instance in instances] [log_intent(ip) for ip in ips] log("")
def ensure_repository(self): try: self.client.create_repository( repositoryName=self.repo_name, imageScanningConfiguration={'scanOnPush': True}, ) log_intent('Repo created with name: ' + self.repo_name) except Exception as ex: if type(ex).__name__ == 'RepositoryAlreadyExistsException': log_intent('Repo exists with name: ' + self.repo_name) else: raise ex current_account_id = get_account_id() if current_account_id != self.account_id: log_intent('Setting cross account ECR access: ' + self.repo_name) self.client.set_repository_policy( repositoryName=self.repo_name, policyText=json.dumps({ "Version": "2008-10-17", "Statement": [{ "Sid": "AllowCrossAccountPull-{}".format(current_account_id), "Effect": "Allow", "Principal": { "AWS": [current_account_id] }, "Action": [ "ecr:GetDownloadUrlForLayer", "ecr:BatchCheckLayerAvailability", "ecr:BatchGetImage", "ecr:InitiateLayerUpload", "ecr:PutImage", "ecr:UploadLayerPart", "ecr:CompleteLayerUpload", ] }] }))
def get_current_image_uri(self): ecr_image_uri = self._fetch_current_image_uri() log_intent(f"Currently deployed tag: {ecr_image_uri}") return str(ecr_image_uri)