def _deploy_user_image(image_name: str, path: str, env: str, build_args: Optional[List[str]]) -> None: region = get_region() account_id = get_account_id() _logger.debug(f"_deploy_user_image args: {account_id}, {region}, {image_name}, {env}") docker.login_v2(account_id=account_id, region=region) _logger.debug("DockerHub and ECR Logged in") _logger.debug("Deploying the %s Docker image", image_name) ecr_repo = f"orbit-{env}/users/{image_name}" if not ecr.describe_repositories(repository_names=[ecr_repo]): ecr.create_repository(repository_name=ecr_repo) _logger.debug("Building and deploy docker image from source...") path = os.path.join(os.getcwd(), image_name) _logger.debug("path: %s", path) docker.deploy_image_from_source( dir=path, name=ecr_repo, build_args=build_args, tag="latest", env=env, ) _logger.debug("Docker Image Deployed to ECR")
def _load_context_from_manifest(manifest: "Manifest") -> Context: _logger.debug("Loading Context from manifest") context_parameter_name: str = f"/orbit/{manifest.name}/context" if ssm.does_parameter_exist(name=context_parameter_name): context: Context = ContextSerDe.load_context_from_ssm( env_name=manifest.name, type=Context) context.networking = create_networking_context_from_manifest( networking=manifest.networking) context.user_pool_id = manifest.user_pool_id context.shared_efs_fs_id = manifest.shared_efs_fs_id context.shared_efs_sg_id = manifest.shared_efs_sg_id context.scratch_bucket_arn = manifest.scratch_bucket_arn context.policies = manifest.policies context.role_prefix = manifest.role_prefix context.cognito_external_provider = manifest.cognito_external_provider context.cognito_external_provider_label = manifest.cognito_external_provider_label else: context = Context( name=manifest.name, account_id=utils.get_account_id(), region=utils.get_region(), env_tag=f"orbit-{manifest.name}", env_stack_name=f"orbit-{manifest.name}", env_ssm_parameter_name=f"/orbit/{manifest.name}/env", eks_stack_name=f"eksctl-orbit-{manifest.name}-cluster", ssm_parameter_name=context_parameter_name, ssm_dockerhub_parameter_name= f"/orbit/{manifest.name}/dockerhub", toolkit=ToolkitManifest( stack_name=f"orbit-{manifest.name}-toolkit", codebuild_project=f"orbit-{manifest.name}", ), cdk_toolkit=CdkToolkitManifest( stack_name=f"orbit-{manifest.name}-cdk-toolkit"), role_prefix=manifest.role_prefix, scratch_bucket_arn=manifest.scratch_bucket_arn, networking=create_networking_context_from_manifest( networking=manifest.networking), images=manifest.images, user_pool_id=manifest.user_pool_id, cognito_external_provider=manifest.cognito_external_provider, cognito_external_provider_label=manifest. cognito_external_provider_label, teams=create_teams_context_from_manifest(manifest=manifest), shared_efs_fs_id=manifest.shared_efs_fs_id, shared_efs_sg_id=manifest.shared_efs_sg_id, managed_nodegroups=[], eks_system_masters_roles=[], policies=manifest.policies, install_ssm_agent=manifest.install_ssm_agent, install_image_replicator=manifest.install_image_replicator, ) context.install_ssm_agent = manifest.install_ssm_agent context.install_image_replicator = manifest.install_image_replicator ContextSerDe.fetch_toolkit_data(context=context) ContextSerDe.dump_context_to_ssm(context=context) return context
def _load_context_from_manifest(manifest: "Manifest") -> Context: _logger.debug("Loading Context from manifest") context_parameter_name: str = f"/orbit/{manifest.name}/context" if ssm.does_parameter_exist(name=context_parameter_name): context: Context = ContextSerDe.load_context_from_ssm( env_name=manifest.name, type=Context) context.images = manifest.images context.networking = create_networking_context_from_manifest( networking=manifest.networking) context.user_pool_id = manifest.user_pool_id context.shared_efs_fs_id = manifest.shared_efs_fs_id context.shared_efs_sg_id = manifest.shared_efs_sg_id context.scratch_bucket_arn = manifest.scratch_bucket_arn context.policies = manifest.policies for team_manifest in manifest.teams: team_context: Optional[TeamContext] = context.get_team_by_name( name=team_manifest.name) if team_context: _logger.debug("Updating context profiles for team %s", team_manifest.name) team_context.profiles = team_manifest.profiles else: context = Context( name=manifest.name, account_id=utils.get_account_id(), region=utils.get_region(), env_tag=f"orbit-{manifest.name}", env_stack_name=f"orbit-{manifest.name}", env_ssm_parameter_name=f"/orbit/{manifest.name}/env", eks_stack_name=f"eksctl-orbit-{manifest.name}-cluster", ssm_parameter_name=context_parameter_name, ssm_dockerhub_parameter_name= f"/orbit/{manifest.name}/dockerhub", toolkit=ToolkitManifest( stack_name=f"orbit-{manifest.name}-toolkit", codebuild_project=f"orbit-{manifest.name}"), cdk_toolkit=CdkToolkitManifest( stack_name=f"orbit-{manifest.name}-cdk-toolkit"), codeartifact_domain=manifest.codeartifact_domain, codeartifact_repository=manifest.codeartifact_repository, scratch_bucket_arn=manifest.scratch_bucket_arn, networking=create_networking_context_from_manifest( networking=manifest.networking), images=manifest.images, user_pool_id=manifest.user_pool_id, cognito_external_provider=manifest.cognito_external_provider, cognito_external_provider_label=manifest. cognito_external_provider_label, teams=create_teams_context_from_manifest(manifest=manifest), shared_efs_fs_id=manifest.shared_efs_fs_id, shared_efs_sg_id=manifest.shared_efs_sg_id, managed_nodegroups=[], eks_system_masters_roles=[], policies=manifest.policies, ) ContextSerDe.fetch_toolkit_data(context=context) ContextSerDe.dump_context_to_ssm(context=context) return context
def write_context_ssm(profiles: PROFILES_TYPE, env_name: str, team_name: str) -> None: ssm_profile_name = f"/orbit/{env_name}/teams/{team_name}/user/profiles" client = utils.boto3_client(service_name="ssm") _logger.debug("Writing team %s user profiles to SSM parameter.", team_name) json_str = str(json.dumps(obj=profiles, sort_keys=True)) # resolve any parameters inside team context per context json_str = utils.resolve_parameters( json_str, dict(region=utils.get_region(), account=utils.get_account_id(), env=env_name, team=team_name) ) client.put_parameter( Name=ssm_profile_name, Value=json_str, Overwrite=True, Tier="Intelligent-Tiering", )
def write_resolve_parameters( manifest_name: str, name: str, filename: str, region: Optional[str], ) -> None: region_str: str = region if region is not None else utils.get_region() input = os.path.join(ORBIT_CLI_ROOT, "data", "init", manifest_name) with open(input, "r") as file: content: str = file.read() content = utils.resolve_parameters( content, dict(region=region_str, name=name, account_id=utils.get_account_id())) with open(filename, "w") as file: file.write(content)
def create_team_context_from_manifest( manifest: "Manifest", team_manifest: "TeamManifest") -> TeamContext: account_id: str = utils.get_account_id() region: str = utils.get_region() ssm_parameter_name: str = f"/orbit/{manifest.name}/teams/{team_manifest.name}/context" version = manifest.images.jupyter_user.version base_image_address = f"{account_id}.dkr.ecr.{region}.amazonaws.com/orbit-{manifest.name}-jupyter-user:{version}" final_image_address = base_image_address return TeamContext( base_image_address=base_image_address, final_image_address=final_image_address, stack_name=f"orbit-{manifest.name}-{team_manifest.name}", ssm_parameter_name=ssm_parameter_name, team_ssm_parameter_name= f"/orbit/{manifest.name}/teams/{team_manifest.name}/team", bootstrap_s3_prefix=f"teams/{manifest.name}/bootstrap/", **team_manifest.__dict__, )
def deploy_image_from_source( dir: str, name: str, env: str, tag: str = "latest", use_cache: bool = True, build_args: Optional[List[str]] = None, ) -> None: _logger.debug( f"deploy_image_from_source {dir} {name} {env} {tag} {build_args}") if not os.path.exists(dir): bundle_dir = os.path.join("bundle", dir) if os.path.exists(bundle_dir): dir = bundle_dir account_id = utils.get_account_id() region = utils.get_region() build_args = [] if build_args is None else build_args _logger.debug("Building docker image from %s", os.path.abspath(dir)) sh.run(cmd="docker system prune --all --force --volumes") update_docker_file(account_id=account_id, region=region, env=env, tag=tag, dir=dir) build( account_id=account_id, region=region, dir=dir, name=name, tag=tag, use_cache=use_cache, pull=True, build_args=build_args, ) _logger.debug("Docker Image built") tag_image(account_id=account_id, region=region, name=name, tag=tag) _logger.debug("Docker Image tagged") push(account_id=account_id, region=region, name=name, tag=tag) _logger.debug("Docker Image pushed")
def _load_toolkit_helper(file_path: str, image_name: str, env: str) -> str: with open(file_path, "r") as file: helper: str = file.read() params = { "ACCOUNT_ID": get_account_id(), "REGION": get_region(), "IMAGE_NAME": image_name, "ENV": env, } t = resolve_parameters(helper, dict(params)) j = json.loads(t) if j.get("extra_dirs"): new_extra_dirs = {} for e in j["extra_dirs"]: key = e val = os.path.realpath(os.path.join(ORBIT_CLI_ROOT, j["extra_dirs"][e])) new_extra_dirs[key] = val _logger.debug(f" new extra dir = {new_extra_dirs}") j["extra_dirs"] = new_extra_dirs _logger.debug(f"OUT of HELPER {j}") return json.dumps(j)
def _load_foundation_context_from_manifest( manifest: "FoundationManifest") -> FoundationContext: _logger.debug("Loading Context from manifest") context_parameter_name: str = f"/orbit-foundation/{manifest.name}/context" if ssm.does_parameter_exist(name=context_parameter_name): context: FoundationContext = ContextSerDe.load_context_from_ssm( env_name=manifest.name, type=FoundationContext) context.images = manifest.images context.policies = manifest.policies context.codeartifact_domain = manifest.codeartifact_domain context.codeartifact_repository = manifest.codeartifact_repository else: context = FoundationContext( name=manifest.name, account_id=utils.get_account_id(), region=utils.get_region(), env_tag=f"orbit-foundation-{manifest.name}", ssm_parameter_name=context_parameter_name, resources_ssm_parameter_name= f"/orbit-foundation/{manifest.name}/resources", toolkit=ToolkitManifest( stack_name=f"orbit-foundation-{manifest.name}-toolkit", codebuild_project=f"orbit-foundation-{manifest.name}", ), cdk_toolkit=CdkToolkitManifest( stack_name=f"orbit-foundation-{manifest.name}-cdk-toolkit" ), codeartifact_domain=manifest.codeartifact_domain, codeartifact_repository=manifest.codeartifact_repository, networking=create_networking_context_from_manifest( networking=manifest.networking), images=manifest.images, policies=manifest.policies, stack_name=f"orbit-foundation-{manifest.name}", ) ContextSerDe.fetch_toolkit_data(context=context) ContextSerDe.dump_context_to_ssm(context=context) return context
def create_team_context_from_manifest( manifest: "Manifest", team_manifest: "TeamManifest") -> TeamContext: account_id: str = utils.get_account_id() region: str = utils.get_region() ssm_parameter_name: str = f"/orbit/{manifest.name}/teams/{team_manifest.name}/context" version = manifest.images.jupyter_user.version _logger.debug("Team Base Image: %s - %s", team_manifest.name, manifest.images.jupyter_user) base_image_address = ( f"{account_id}.dkr.ecr.{region}.amazonaws.com/orbit-{manifest.name}/jupyter-user:{version}" if manifest.images.jupyter_user.get_source(account_id=account_id, region=region) == "code" else f"{manifest.images.jupyter_user.repository}:{version}") final_image_address = base_image_address return TeamContext( base_image_address=base_image_address, final_image_address=final_image_address, stack_name=f"orbit-{manifest.name}-{team_manifest.name}", ssm_parameter_name=ssm_parameter_name, team_ssm_parameter_name= f"/orbit/{manifest.name}/teams/{team_manifest.name}/team", bootstrap_s3_prefix=f"teams/{manifest.name}/bootstrap/", **team_manifest.__dict__, )