def _fetch_current_image_uri(self): ecs_client = get_client_for('ecs', self.environment) if len(self.service_info) < 1: raise UnrecoverableException( "cannot get running image_uri: no ECS services found") ecs_service_name = next( service_info['ecs_service_name'] for service, service_info in self.service_info.items() if service_info.get('ecs_service_name')) ecs_services = ecs_client.describe_services( cluster=self.cluster_name, services=[ecs_service_name], )['services'] if len(ecs_services) < 1: raise UnrecoverableException( "cannot get running image_uri: no service found.") task_definition_arn = ecs_services[0]['taskDefinition'] task_definition = ecs_client.describe_task_definition( taskDefinition=task_definition_arn) return task_definition['taskDefinition']['containerDefinitions'][0][ 'image']
def _fetch_current_image_uri(self): ecs_client = get_client_for('ecs', self.environment) if len(self.service_info) < 1: raise UnrecoverableException( "cannot get running image_uri: no ECS services found") logical_service_name = next(iter(self.service_info)) ecs_service_name = self.service_info[logical_service_name].get( 'ecs_service_name') task_arns = ecs_client.list_tasks( cluster=self.cluster_name, serviceName=ecs_service_name)['taskArns'] if len(task_arns) < 1: raise UnrecoverableException( "cannot get running image_uri: no task ARNs found for service") tasks = ecs_client.describe_tasks(cluster=self.cluster_name, tasks=task_arns)['tasks'] task_definition_arns = tasks[0]['taskDefinitionArn'] task_definition = ecs_client.describe_task_definition( taskDefinition=task_definition_arns) return task_definition['taskDefinition']['containerDefinitions'][0][ 'image']
def get_config(self, cloudlift_version=VERSION): ''' Get configuration from DynamoDB ''' try: configuration_response = self.table.get_item( Key={ 'environment': self.environment }, ConsistentRead=True, AttributesToGet=[ 'configuration' ] ) existing_configuration = configuration_response['Item']['configuration'] previous_cloudlift_version = existing_configuration.pop("cloudlift_version", None) # print(f"Previous cloudlift version in environment config is {previous_cloudlift_version}") if previous_cloudlift_version and LooseVersion(cloudlift_version) < LooseVersion(previous_cloudlift_version): raise UnrecoverableException(f'Cloudlift Version {previous_cloudlift_version} was used to ' f'create this service. You are using version {cloudlift_version}, ' f'which is older and can cause corruption. Please upgrade to at least ' f'version {previous_cloudlift_version} to proceed.\n\nUpgrade to the ' f'latest version (Recommended):\n' f'\tpip install -U cloudlift\n\nOR\n\nUpgrade to a compatible version:\n' f'\tpip install -U cloudlift=={previous_cloudlift_version}') return existing_configuration except ClientError: raise UnrecoverableException("Unable to fetch environment configuration from DynamoDB.") except KeyError: raise UnrecoverableException("Environment configuration not found. Does this environment exist?")
def get_client_for(resource, environment): try: return boto3.session.Session( region_name=get_region_for_environment(environment) ).client(resource) except ClientError as error: if error.response['Error']['Code'] == 'ExpiredTokenException': raise UnrecoverableException("AWS session associated with this profile has expired or is otherwise invalid") elif error.response['Error']['Code'] == 'InvalidIdentityTokenException': raise UnrecoverableException("AWS token that was passed could not be validated by Amazon Web Services") elif error.response['Error']['Code'] == 'RegionDisabledException': raise UnrecoverableException("STS is not activated in the requested region for the account that is being asked to generate credentials") else: raise UnrecoverableException("Unable to find valid AWS credentials")
def generate_service(self): self._add_service_parameters() self._add_service_outputs() self._fetch_current_desired_count() self._add_ecs_service_iam_role() self._add_cluster_services() key = uuid.uuid4().hex + '.yml' if len(to_yaml(self.template.to_json())) > 51000: try: self.client.put_object( Body=to_yaml(self.template.to_json()), Bucket=self.bucket_name, Key=key, ) template_url = f'https://{self.bucket_name}.s3.amazonaws.com/{key}' return template_url, 'TemplateURL', key except ClientError as boto_client_error: error_code = boto_client_error.response['Error']['Code'] if error_code == 'AccessDenied': raise UnrecoverableException( f'Unable to store cloudlift service template in S3 bucket at {self.bucket_name}' ) else: raise boto_client_error else: return to_yaml(self.template.to_json()), 'TemplateBody', ''
def create(self): ''' Create and execute CloudFormation template for ECS service and related dependencies ''' log_bold("Initiating service creation") self.service_configuration.edit_config() template_generator = ServiceTemplateGenerator( self.service_configuration, self.environment_stack) service_template_body = template_generator.generate_service() try: self.client.create_stack( StackName=self.stack_name, TemplateBody=service_template_body, Parameters=[{ 'ParameterKey': 'Environment', 'ParameterValue': self.environment, }], OnFailure='DO_NOTHING', Capabilities=['CAPABILITY_NAMED_IAM'], ) log_bold("Submitted to cloudformation. Checking progress...") self._print_progress() except ClientError as boto_client_error: error_code = boto_client_error.response['Error']['Code'] if error_code == 'AlreadyExistsException': raise UnrecoverableException("Stack " + self.stack_name + " already exists.") else: raise boto_client_error
def create(self, config_body=None, version=None, build_arg=None, dockerfile=None, ssh=None, cache_from=None): ''' Create and execute CloudFormation template for ECS service and related dependencies ''' log_bold("Initiating service creation") if config_body is None: self.service_configuration.edit_config() else: self.service_configuration.set_config(config_body) self.service_configuration.validate() ecr_repo_config = self.service_configuration.get_config().get( 'ecr_repo') ecr = ECR( region=get_region_for_environment(self.environment), repo_name=ecr_repo_config.get('name'), account_id=ecr_repo_config.get('account_id', None), assume_role_arn=ecr_repo_config.get('assume_role_arn', None), version=version, build_args=build_arg, dockerfile=dockerfile, ssh=ssh, cache_from=cache_from, ) ecr.upload_artefacts() template_generator = ServiceTemplateGenerator( self.service_configuration, self.environment_stack, self.env_sample_file, ecr.image_uri) service_template_body = template_generator.generate_service() try: options = prepare_stack_options_for_template( service_template_body, self.environment, self.stack_name) self.client.create_stack( StackName=self.stack_name, Parameters=[{ 'ParameterKey': 'Environment', 'ParameterValue': self.environment, }], OnFailure='DO_NOTHING', Capabilities=['CAPABILITY_NAMED_IAM'], **options, ) log_bold("Submitted to cloudformation. Checking progress...") self._print_progress() except ClientError as boto_client_error: error_code = boto_client_error.response['Error']['Code'] if error_code == 'AlreadyExistsException': raise UnrecoverableException("Stack " + self.stack_name + " already exists.") else: raise boto_client_error
def _initiate_session(self, target_instance): log_bold("Starting session in " + target_instance) try: driver = create_clidriver() driver.main(["ssm", "start-session", "--target", target_instance]) except: raise UnrecoverableException("Failed to start session")
def __get_desired_count(self): try: auto_scaling_client = get_client_for( 'autoscaling', self.environment ) cloudformation_client = get_client_for( 'cloudformation', self.environment ) cfn_resources = cloudformation_client.list_stack_resources( StackName=self.cluster_name ) auto_scaling_group_name = list( filter( lambda x: x['ResourceType'] == "AWS::AutoScaling::AutoScalingGroup", cfn_resources['StackResourceSummaries'] ) )[0]['PhysicalResourceId'] response = auto_scaling_client.describe_auto_scaling_groups( AutoScalingGroupNames=[auto_scaling_group_name] ) return response['AutoScalingGroups'][0]['DesiredCapacity'] except Exception: raise UnrecoverableException("Unable to fetch desired instance count.")
def get_ssl_certification_for_environment(environment): try: return EnvironmentConfiguration( environment ).get_config()[environment]['environment']["ssl_certificate_arn"] except KeyError: raise UnrecoverableException("Unable to find ssl certificate for {environment}".format(**locals()))
def get_environment_level_alb_listener(environment): env_spec = EnvironmentConfiguration(environment).get_config()[environment] if 'loadbalancer_listener_arn' not in env_spec: raise UnrecoverableException('environment level ALB not defined. ' + 'Please run update_environment and set "loadbalancer_listener_arn".') return env_spec['loadbalancer_listener_arn']
def _edit_config(self): ''' Open editor to update configuration ''' try: current_configuration = self.get_config() updated_configuration = edit( json.dumps( current_configuration, indent=4, sort_keys=True, cls=DecimalEncoder ) ) if updated_configuration is None: log_warning("No changes made.") else: updated_configuration = json.loads(updated_configuration) differences = list(dictdiffer.diff( current_configuration, updated_configuration )) if not differences: log_warning("No changes made.") else: print_json_changes(differences) if confirm('Do you want update the config?'): self._set_config(updated_configuration) else: log_warning("Changes aborted.") except ClientError: raise UnrecoverableException("Unable to fetch environment configuration from DynamoDB.")
def get_config(self): ''' Get configuration from DynamoDB ''' try: configuration_response = self.table.get_item( Key={ 'service_name': self.service_name, 'environment': self.environment }, ConsistentRead=True, AttributesToGet=['configuration']) if 'Item' in configuration_response: existing_configuration = configuration_response['Item'][ 'configuration'] else: existing_configuration = self._default_service_configuration() self.new_service = True existing_configuration.pop("cloudlift_version", None) return existing_configuration except ClientError: raise UnrecoverableException( "Unable to fetch service configuration from DynamoDB.")
def set_version(self, version): if version: try: commit_sha = self._find_commit_sha(version) except: commit_sha = version log_intent("Using commit hash " + commit_sha + " to find image") image = self._find_image_in_ecr(commit_sha) if not image: log_warning("Please build, tag and upload the image for the \ commit " + commit_sha) raise UnrecoverableException( "Image for given version could not be found.") self.version = version else: dirty = subprocess.check_output(["git", "status", "--short"]).decode("utf-8") if dirty: self.version = 'dirty' log_intent("Version parameter was not provided. Determined \ version to be " + self.version + " based on current status") else: self.version = self._find_commit_sha() log_intent("Version parameter was not provided. Determined \ version to be " + self.version + " based on current status")
def __init__(self, name, environment='', env_sample_file='', timeout_seconds=None, version=None, build_args=None, dockerfile=None, ssh=None, cache_from=None, deployment_identifier=None, working_dir='.'): self.name = name self.environment = environment self.deployment_identifier = deployment_identifier self.env_sample_file = env_sample_file self.timeout_seconds = timeout_seconds self.version = version self.ecr_client = boto3.session.Session(region_name=self.region).client('ecr') self.cluster_name = get_cluster_name(environment) self.service_configuration = ServiceConfiguration(service_name=name, environment=environment).get_config() self.service_info_fetcher = ServiceInformationFetcher(self.name, self.environment, self.service_configuration) if not self.service_info_fetcher.stack_found: raise UnrecoverableException( "error finding stack in ServiceUpdater: {}-{}".format(self.name, self.environment)) ecr_repo_config = self.service_configuration.get('ecr_repo') self.ecr = ECR( self.region, ecr_repo_config.get('name', spinalcase(self.name + '-repo')), ecr_repo_config.get('account_id', get_account_id()), ecr_repo_config.get('assume_role_arn', None), version, build_args, dockerfile, working_dir, ssh, cache_from )
def build_secrets_for_all_namespaces(env_name, service_name, ecs_service_name, sample_env_folder_path, secrets_name): secrets_across_namespaces = {} namespaces = get_namespaces_from_directory(sample_env_folder_path) duplicates = find_duplicate_keys(sample_env_folder_path, namespaces) if len(duplicates) != 0: raise UnrecoverableException( 'duplicate keys found in env sample files {} '.format(duplicates)) for namespace in namespaces: secrets_for_namespace = _get_secrets_for_namespace( env_name, namespace, sample_env_folder_path, secrets_name) secrets_across_namespaces.update(secrets_for_namespace) automated_secret_name = get_automated_injected_secret_name( env_name, service_name, ecs_service_name) existing_secrets = {} try: existing_secrets = secrets_manager.get_config(automated_secret_name, env_name)['secrets'] except Exception as err: log_warning( f'secret {automated_secret_name} does not exist. It will be created: {err}' ) if existing_secrets != secrets_across_namespaces: log(f"Updating {automated_secret_name}") secrets_manager.set_secrets_manager_config(env_name, automated_secret_name, secrets_across_namespaces) arn = secrets_manager.get_config(automated_secret_name, env_name)['ARN'] return dict(CLOUDLIFT_INJECTED_SECRETS=arn)
def _get_random_available_listener_rule_priority(listener_rules, listener_arn): occupied_priorities = set(rule['Priority'] for rule in listener_rules) available_priorities = set(range(1, 50001)) - occupied_priorities if not available_priorities: raise UnrecoverableException( "No listener rule priorities available for listener_arn: {}". format(listener_arn)) return int(random.choice(list(available_priorities)))
def _push_image(self, local_name, ecr_name): try: subprocess.check_call(["docker", "tag", local_name, ecr_name]) except: raise UnrecoverableException("Local image was not found.") self._login_to_ecr() subprocess.check_call(["docker", "push", ecr_name]) subprocess.check_call(["docker", "rmi", ecr_name]) log_intent('Pushed the image (' + local_name + ') to ECR sucessfully.')
def get_config(self): ''' Get configuration from DynamoDB ''' try: configuration_response = self.table.get_item( Key={'environment': self.environment}, ConsistentRead=True, AttributesToGet=['configuration']) return configuration_response['Item']['configuration'] except ClientError: raise UnrecoverableException( "Unable to fetch environment configuration from DynamoDB.") except KeyError: raise UnrecoverableException( "Environment configuration not found. Does this environment exist?" )
def _get_target_instance(self): service_instance_ids = ServiceInformationFetcher( self.name, self.environment).get_instance_ids() if not service_instance_ids: raise UnrecoverableException("Couldn't find instances. Exiting.") instance_ids = list( set(functools.reduce(operator.add, service_instance_ids.values()))) log("Found " + str(len(instance_ids)) + " instances to start session") return instance_ids[0]
def _get_parameter_store_config(service_name, env_name): try: environment_config, _ = ParameterStore( service_name, env_name).get_existing_config_paths() except Exception as err: log_intent(str(err)) ex_msg = f"Cannot find the configuration in parameter store [env: ${env_name} | service: ${service_name}]." raise UnrecoverableException(ex_msg) return environment_config
def run(self): log_warning("Deploying to {self.region}".format(**locals())) self.init_stack_info() if not os.path.exists(self.env_sample_file): raise UnrecoverableException('env.sample not found. Exiting.') log_intent("name: " + self.name + " | environment: " + self.environment + " | version: " + str(self.version)) log_bold("Checking image in ECR") self.upload_artefacts() log_bold("Initiating deployment\n") ecs_client = EcsClient(None, None, self.region) jobs = [] for index, service_name in enumerate(self.ecs_service_names): log_bold("Starting to deploy " + service_name) color = DEPLOYMENT_COLORS[index % 3] image_url = self.ecr_image_uri image_url += (':' + self.version) process = multiprocessing.Process( target=deployer.deploy_new_version, args=( ecs_client, self.cluster_name, service_name, self.version, self.name, self.env_sample_file, self.environment, color, image_url ) ) jobs.append(process) process.start() exit_codes = [] while True: sleep(1) exit_codes = [proc.exitcode for proc in jobs] if None not in exit_codes: break if any(exit_codes) != 0: raise UnrecoverableException("Deployment failed")
def verify_env_sample(self, env_sample_directory_path): if not self.stack_found: raise UnrecoverableException( "error finding stack in ServiceUpdater: {}-{}".format( self.name, self.environment)) service_info = self.service_info for ecs_service_logical_name in service_info: ecs_service_info = service_info[ecs_service_logical_name] secrets_name = ecs_service_info.get('secrets_name') verify_and_get_secrets_for_all_namespaces( self.environment, env_sample_directory_path, secrets_name)
def build_config(env_name, service_name, sample_env_file_path): service_config = read_config(open(sample_env_file_path).read()) try: environment_config = ParameterStore(service_name, env_name).get_existing_config() except Exception as err: log_intent(str(err)) raise UnrecoverableException( "Cannot find the configuration in parameter store \ [env: %s | service: %s]." % (env_name, service_name)) missing_env_config = set(service_config) - set(environment_config) if missing_env_config: raise UnrecoverableException('There is no config value for the keys ' + str(missing_env_config)) missing_env_sample_config = set(environment_config) - set(service_config) if missing_env_sample_config: raise UnrecoverableException( 'There is no config value for the keys in env.sample file ' + str(missing_env_sample_config)) return make_container_defn_env_conf(service_config, environment_config)
def _find_commit_sha(self, version=None): log_intent("Finding commit SHA") try: version_to_find = version or "HEAD" commit_sha = subprocess.check_output( ["git", "rev-list", "-n", "1", version_to_find] ).strip().decode("utf-8") log_intent("Found commit SHA " + commit_sha) return commit_sha except: raise UnrecoverableException("Commit SHA not found. Given version is not a git tag, \ branch or commit SHA")
def _get_environment_stack(self): try: log("Looking for " + self.environment + " cluster.") environment_stack = self.client.describe_stacks( StackName=get_cluster_name(self.environment))['Stacks'][0] log_bold(self.environment + " stack found. Using stack with ID: " + environment_stack['StackId']) except ClientError: raise UnrecoverableException( self.environment + " cluster not found. Create the environment \ cluster using `create_environment` command.") return environment_stack
def get_config(self, cloudlift_version): ''' Get configuration from DynamoDB ''' try: configuration_response = self.table.get_item( Key={ 'service_name': self.service_name, 'environment': self.environment }, ConsistentRead=True, AttributesToGet=['configuration']) if 'Item' in configuration_response: existing_configuration = configuration_response['Item'][ 'configuration'] from distutils.version import LooseVersion previous_cloudlift_version = existing_configuration.pop( "cloudlift_version", None) if LooseVersion(cloudlift_version) < LooseVersion( previous_cloudlift_version): raise UnrecoverableException( f'Cloudlift Version {previous_cloudlift_version} was used to ' f'create this service. You are using version {cloudlift_version}, ' f'which is older and can cause corruption. Please upgrade to at least ' f'version {previous_cloudlift_version} to proceed.\n\nUpgrade to the ' f'latest version (Recommended):\n' f'\tpip install -U cloudlift\n\nOR\n\nUpgrade to a compatible version:\n' f'\tpip install -U cloudlift=={previous_cloudlift_version}' ) else: existing_configuration = self._default_service_configuration() self.new_service = True return existing_configuration except ClientError: raise UnrecoverableException( "Unable to fetch service configuration from DynamoDB.")
def verify_and_get_secrets_for_all_namespaces(env_name, sample_env_folder_path, secrets_name): secrets_across_namespaces = {} namespaces = get_namespaces_from_directory(sample_env_folder_path) duplicates = find_duplicate_keys(sample_env_folder_path, namespaces) if len(duplicates) != 0: raise UnrecoverableException( 'duplicate keys found in env sample files {} '.format(duplicates)) for namespace in namespaces: secrets_for_namespace = _get_secrets_for_namespace( env_name, namespace, sample_env_folder_path, secrets_name) secrets_across_namespaces.update(secrets_for_namespace) return secrets_across_namespaces
def get_mfa_session(mfa_code=None, region='ap-south-1'): username = get_username() if not mfa_code: mfa_code = input("MFA Code: ") mfa_arn = "arn:aws:iam::%s:mfa/%s" % (get_account_id(), username) log_bold("Using credentials for " + username) try: session_params = client('sts').get_session_token( DurationSeconds=900, SerialNumber=mfa_arn, TokenCode=str(mfa_code)) credentials = session_params['Credentials'] return Session(aws_access_key_id=credentials['AccessKeyId'], aws_secret_access_key=credentials['SecretAccessKey'], aws_session_token=credentials['SessionToken'], region_name=region) except botocore.exceptions.ClientError as client_error: raise UnrecoverableException(str(client_error))
def get_config_in_db(self): ''' Get configuration from DynamoDB ''' try: configuration_response = self.table.get_item( Key={k: v for k, v in self.kv_pairs}, ConsistentRead=True, AttributesToGet=[ 'configuration' ] ) if 'Item' in configuration_response: return configuration_response['Item']['configuration'] return None except ClientError: raise UnrecoverableException("Unable to fetch configuration from DynamoDB.")