def manifest_validations(manifest: Manifest) -> None:
for managed_nodegroup in manifest.managed_nodegroups:
_logger.debug(f"Validating managed node group {managed_nodegroup} ")
if managed_nodegroup.nodes_num_desired < 1:
raise ValueError(
f"{managed_nodegroup.name} number of desired nodes should be greater than 0"
)
# Handle IAM role name 64 character limit validation
region_len = len(utils.get_region())
env_len = len(manifest.name)
env_region_len = env_len + region_len
_logger.debug(f"env_region_len={env_region_len}")
env_region_max = 28
env_region_team_max = 51
if env_region_len > env_region_max:
raise ValueError(
f"Orbit accepts max combined length of {env_region_max} characters for env and region"
)
max_team_len = max([len(team.name) for team in manifest.teams])
env_region_team_len = env_region_len + max_team_len
_logger.debug(f"env_region_team_len={env_region_team_len}")
if env_region_team_len > env_region_team_max:
raise ValueError(
f"Orbit accepts max combined length of {env_region_team_max} characters for env, region, and team name"
)
def _deploy_user_image(image_name: str, path: str, env: str, build_args: Optional[List[str]]) -> None:
region = get_region()
account_id = get_account_id()
_logger.debug(f"_deploy_user_image args: {account_id}, {region}, {image_name}, {env}")
docker.login_v2(account_id=account_id, region=region)
_logger.debug("DockerHub and ECR Logged in")
_logger.debug("Deploying the %s Docker image", image_name)
ecr_repo = f"orbit-{env}/users/{image_name}"
if not ecr.describe_repositories(repository_names=[ecr_repo]):
ecr.create_repository(repository_name=ecr_repo)
_logger.debug("Building and deploy docker image from source...")
path = os.path.join(os.getcwd(), image_name)
_logger.debug("path: %s", path)
docker.deploy_image_from_source(
dir=path,
name=ecr_repo,
build_args=build_args,
tag="latest",
env=env,
)
_logger.debug("Docker Image Deployed to ECR")
def _load_context_from_manifest(manifest: "Manifest") -> Context:
_logger.debug("Loading Context from manifest")
context_parameter_name: str = f"/orbit/{manifest.name}/context"
if ssm.does_parameter_exist(name=context_parameter_name):
context: Context = ContextSerDe.load_context_from_ssm(
env_name=manifest.name, type=Context)
context.networking = create_networking_context_from_manifest(
networking=manifest.networking)
context.user_pool_id = manifest.user_pool_id
context.shared_efs_fs_id = manifest.shared_efs_fs_id
context.shared_efs_sg_id = manifest.shared_efs_sg_id
context.scratch_bucket_arn = manifest.scratch_bucket_arn
context.policies = manifest.policies
context.role_prefix = manifest.role_prefix
context.cognito_external_provider = manifest.cognito_external_provider
context.cognito_external_provider_label = manifest.cognito_external_provider_label
else:
context = Context(
name=manifest.name,
account_id=utils.get_account_id(),
region=utils.get_region(),
env_tag=f"orbit-{manifest.name}",
env_stack_name=f"orbit-{manifest.name}",
env_ssm_parameter_name=f"/orbit/{manifest.name}/env",
eks_stack_name=f"eksctl-orbit-{manifest.name}-cluster",
ssm_parameter_name=context_parameter_name,
ssm_dockerhub_parameter_name=
f"/orbit/{manifest.name}/dockerhub",
toolkit=ToolkitManifest(
stack_name=f"orbit-{manifest.name}-toolkit",
codebuild_project=f"orbit-{manifest.name}",
),
cdk_toolkit=CdkToolkitManifest(
stack_name=f"orbit-{manifest.name}-cdk-toolkit"),
role_prefix=manifest.role_prefix,
scratch_bucket_arn=manifest.scratch_bucket_arn,
networking=create_networking_context_from_manifest(
networking=manifest.networking),
images=manifest.images,
user_pool_id=manifest.user_pool_id,
cognito_external_provider=manifest.cognito_external_provider,
cognito_external_provider_label=manifest.
cognito_external_provider_label,
teams=create_teams_context_from_manifest(manifest=manifest),
shared_efs_fs_id=manifest.shared_efs_fs_id,
shared_efs_sg_id=manifest.shared_efs_sg_id,
managed_nodegroups=[],
eks_system_masters_roles=[],
policies=manifest.policies,
install_ssm_agent=manifest.install_ssm_agent,
install_image_replicator=manifest.install_image_replicator,
)
context.install_ssm_agent = manifest.install_ssm_agent
context.install_image_replicator = manifest.install_image_replicator
ContextSerDe.fetch_toolkit_data(context=context)
ContextSerDe.dump_context_to_ssm(context=context)
return context
def _load_context_from_manifest(manifest: "Manifest") -> Context:
_logger.debug("Loading Context from manifest")
context_parameter_name: str = f"/orbit/{manifest.name}/context"
if ssm.does_parameter_exist(name=context_parameter_name):
context: Context = ContextSerDe.load_context_from_ssm(
env_name=manifest.name, type=Context)
context.images = manifest.images
context.networking = create_networking_context_from_manifest(
networking=manifest.networking)
context.user_pool_id = manifest.user_pool_id
context.shared_efs_fs_id = manifest.shared_efs_fs_id
context.shared_efs_sg_id = manifest.shared_efs_sg_id
context.scratch_bucket_arn = manifest.scratch_bucket_arn
context.policies = manifest.policies
for team_manifest in manifest.teams:
team_context: Optional[TeamContext] = context.get_team_by_name(
name=team_manifest.name)
if team_context:
_logger.debug("Updating context profiles for team %s",
team_manifest.name)
team_context.profiles = team_manifest.profiles
else:
context = Context(
name=manifest.name,
account_id=utils.get_account_id(),
region=utils.get_region(),
env_tag=f"orbit-{manifest.name}",
env_stack_name=f"orbit-{manifest.name}",
env_ssm_parameter_name=f"/orbit/{manifest.name}/env",
eks_stack_name=f"eksctl-orbit-{manifest.name}-cluster",
ssm_parameter_name=context_parameter_name,
ssm_dockerhub_parameter_name=
f"/orbit/{manifest.name}/dockerhub",
toolkit=ToolkitManifest(
stack_name=f"orbit-{manifest.name}-toolkit",
codebuild_project=f"orbit-{manifest.name}"),
cdk_toolkit=CdkToolkitManifest(
stack_name=f"orbit-{manifest.name}-cdk-toolkit"),
codeartifact_domain=manifest.codeartifact_domain,
codeartifact_repository=manifest.codeartifact_repository,
scratch_bucket_arn=manifest.scratch_bucket_arn,
networking=create_networking_context_from_manifest(
networking=manifest.networking),
images=manifest.images,
user_pool_id=manifest.user_pool_id,
cognito_external_provider=manifest.cognito_external_provider,
cognito_external_provider_label=manifest.
cognito_external_provider_label,
teams=create_teams_context_from_manifest(manifest=manifest),
shared_efs_fs_id=manifest.shared_efs_fs_id,
shared_efs_sg_id=manifest.shared_efs_sg_id,
managed_nodegroups=[],
eks_system_masters_roles=[],
policies=manifest.policies,
)
ContextSerDe.fetch_toolkit_data(context=context)
ContextSerDe.dump_context_to_ssm(context=context)
return context
def write_resolve_parameters(
manifest_name: str,
name: str,
filename: str,
region: Optional[str],
) -> None:
region_str: str = region if region is not None else utils.get_region()
input = os.path.join(ORBIT_CLI_ROOT, "data", "init", manifest_name)
with open(input, "r") as file:
content: str = file.read()
content = utils.resolve_parameters(content,
dict(region=region_str, name=name))
with open(filename, "w") as file:
file.write(content)
def write_context_ssm(profiles: PROFILES_TYPE, env_name: str, team_name: str) -> None:
ssm_profile_name = f"/orbit/{env_name}/teams/{team_name}/user/profiles"
client = utils.boto3_client(service_name="ssm")
_logger.debug("Writing team %s user profiles to SSM parameter.", team_name)
json_str = str(json.dumps(obj=profiles, sort_keys=True))
# resolve any parameters inside team context per context
json_str = utils.resolve_parameters(
json_str, dict(region=utils.get_region(), account=utils.get_account_id(), env=env_name, team=team_name)
)
client.put_parameter(
Name=ssm_profile_name,
Value=json_str,
Overwrite=True,
Tier="Intelligent-Tiering",
)
def create_team_context_from_manifest(
manifest: "Manifest", team_manifest: "TeamManifest") -> TeamContext:
account_id: str = utils.get_account_id()
region: str = utils.get_region()
ssm_parameter_name: str = f"/orbit/{manifest.name}/teams/{team_manifest.name}/context"
version = manifest.images.jupyter_user.version
base_image_address = f"{account_id}.dkr.ecr.{region}.amazonaws.com/orbit-{manifest.name}-jupyter-user:{version}"
final_image_address = base_image_address
return TeamContext(
base_image_address=base_image_address,
final_image_address=final_image_address,
stack_name=f"orbit-{manifest.name}-{team_manifest.name}",
ssm_parameter_name=ssm_parameter_name,
team_ssm_parameter_name=
f"/orbit/{manifest.name}/teams/{team_manifest.name}/team",
bootstrap_s3_prefix=f"teams/{manifest.name}/bootstrap/",
**team_manifest.__dict__,
)
def deploy_image_from_source(
dir: str,
name: str,
env: str,
tag: str = "latest",
use_cache: bool = True,
build_args: Optional[List[str]] = None,
) -> None:
_logger.debug(
f"deploy_image_from_source {dir} {name} {env} {tag} {build_args}")
if not os.path.exists(dir):
bundle_dir = os.path.join("bundle", dir)
if os.path.exists(bundle_dir):
dir = bundle_dir
account_id = utils.get_account_id()
region = utils.get_region()
build_args = [] if build_args is None else build_args
_logger.debug("Building docker image from %s", os.path.abspath(dir))
sh.run(cmd="docker system prune --all --force --volumes")
update_docker_file(account_id=account_id,
region=region,
env=env,
tag=tag,
dir=dir)
build(
account_id=account_id,
region=region,
dir=dir,
name=name,
tag=tag,
use_cache=use_cache,
pull=True,
build_args=build_args,
)
_logger.debug("Docker Image built")
tag_image(account_id=account_id, region=region, name=name, tag=tag)
_logger.debug("Docker Image tagged")
push(account_id=account_id, region=region, name=name, tag=tag)
_logger.debug("Docker Image pushed")
def _load_toolkit_helper(file_path: str, image_name: str, env: str) -> str:
with open(file_path, "r") as file:
helper: str = file.read()
params = {
"ACCOUNT_ID": get_account_id(),
"REGION": get_region(),
"IMAGE_NAME": image_name,
"ENV": env,
}
t = resolve_parameters(helper, dict(params))
j = json.loads(t)
if j.get("extra_dirs"):
new_extra_dirs = {}
for e in j["extra_dirs"]:
key = e
val = os.path.realpath(os.path.join(ORBIT_CLI_ROOT, j["extra_dirs"][e]))
new_extra_dirs[key] = val
_logger.debug(f" new extra dir = {new_extra_dirs}")
j["extra_dirs"] = new_extra_dirs
_logger.debug(f"OUT of HELPER {j}")
return json.dumps(j)
def _load_foundation_context_from_manifest(
manifest: "FoundationManifest") -> FoundationContext:
_logger.debug("Loading Context from manifest")
context_parameter_name: str = f"/orbit-foundation/{manifest.name}/context"
if ssm.does_parameter_exist(name=context_parameter_name):
context: FoundationContext = ContextSerDe.load_context_from_ssm(
env_name=manifest.name, type=FoundationContext)
context.images = manifest.images
context.policies = manifest.policies
context.codeartifact_domain = manifest.codeartifact_domain
context.codeartifact_repository = manifest.codeartifact_repository
else:
context = FoundationContext(
name=manifest.name,
account_id=utils.get_account_id(),
region=utils.get_region(),
env_tag=f"orbit-foundation-{manifest.name}",
ssm_parameter_name=context_parameter_name,
resources_ssm_parameter_name=
f"/orbit-foundation/{manifest.name}/resources",
toolkit=ToolkitManifest(
stack_name=f"orbit-foundation-{manifest.name}-toolkit",
codebuild_project=f"orbit-foundation-{manifest.name}",
),
cdk_toolkit=CdkToolkitManifest(
stack_name=f"orbit-foundation-{manifest.name}-cdk-toolkit"
),
codeartifact_domain=manifest.codeartifact_domain,
codeartifact_repository=manifest.codeartifact_repository,
networking=create_networking_context_from_manifest(
networking=manifest.networking),
images=manifest.images,
policies=manifest.policies,
stack_name=f"orbit-foundation-{manifest.name}",
)
ContextSerDe.fetch_toolkit_data(context=context)
ContextSerDe.dump_context_to_ssm(context=context)
return context
def create_team_context_from_manifest(
manifest: "Manifest", team_manifest: "TeamManifest") -> TeamContext:
account_id: str = utils.get_account_id()
region: str = utils.get_region()
ssm_parameter_name: str = f"/orbit/{manifest.name}/teams/{team_manifest.name}/context"
version = manifest.images.jupyter_user.version
_logger.debug("Team Base Image: %s - %s", team_manifest.name,
manifest.images.jupyter_user)
base_image_address = (
f"{account_id}.dkr.ecr.{region}.amazonaws.com/orbit-{manifest.name}/jupyter-user:{version}"
if manifest.images.jupyter_user.get_source(account_id=account_id,
region=region) == "code"
else f"{manifest.images.jupyter_user.repository}:{version}")
final_image_address = base_image_address
return TeamContext(
base_image_address=base_image_address,
final_image_address=final_image_address,
stack_name=f"orbit-{manifest.name}-{team_manifest.name}",
ssm_parameter_name=ssm_parameter_name,
team_ssm_parameter_name=
f"/orbit/{manifest.name}/teams/{team_manifest.name}/team",
bootstrap_s3_prefix=f"teams/{manifest.name}/bootstrap/",
**team_manifest.__dict__,
)
