Esempio n. 1
0
def map_iam_identities(
    context: Context, cluster_name: str, eks_system_masters_roles_changes: Optional[ListChangeset]
) -> None:
    if eks_system_masters_roles_changes and eks_system_masters_roles_changes.added_values:
        for role in eks_system_masters_roles_changes.added_values:
            if iam.get_role(role) is None:
                _logger.debug(f"Skipping nonexisting IAM Role: {role}")
                continue

            arn = f"arn:aws:iam::{context.account_id}:role/{role}"
            for line in sh.run_iterating(f"eksctl get iamidentitymapping --cluster {cluster_name} --arn {arn}"):
                if line.startswith("Error: no iamidentitymapping with arn"):
                    _logger.debug(f"Adding IAM Identity Mapping - Role: {arn}, Username: {role}, Group: system:masters")
                    sh.run(
                        f"eksctl create iamidentitymapping --cluster {cluster_name} --arn {arn} "
                        f"--username {role} --group system:masters"
                    )
                    cast(List[str], context.eks_system_masters_roles).append(role)
                    ContextSerDe.dump_context_to_ssm(context=context)
                    break
            else:
                _logger.debug(f"Skip adding existing IAM Identity Mapping - Role: {arn}")

    if eks_system_masters_roles_changes and eks_system_masters_roles_changes.removed_values:
        for role in eks_system_masters_roles_changes.removed_values:
            arn = f"arn:aws:iam::{context.account_id}:role/{role}"
            for line in sh.run_iterating(f"eksctl get iamidentitymapping --cluster {cluster_name} --arn {arn}"):
                if line.startswith("Error: no iamidentitymapping with arn"):
                    _logger.debug(f"Skip removing nonexisting IAM Identity Mapping - Role: {arn}")
                    break
            else:
                _logger.debug(f"Removing IAM Identity Mapping - Role: {arn}")
                sh.run(f"eksctl delete iamidentitymapping --cluster {cluster_name} --arn {arn} --all")
                cast(List[str], context.eks_system_masters_roles).remove(role)
                ContextSerDe.dump_context_to_ssm(context=context)
def fetch_kubectl_data(context: "Context", k8s_context: str,
                       include_teams: bool) -> None:
    _logger.debug("Fetching Kubectl data...")

    if include_teams:
        for team in context.teams:
            _logger.debug("Fetching team %s URL parameter", team.name)
            url = k8s.get_service_hostname(name="jupyterhub-public",
                                           k8s_context=k8s_context,
                                           namespace=team.name)
            team.jupyter_url = url

    landing_page_url: str = k8s.get_service_hostname(name="landing-page",
                                                     k8s_context=k8s_context,
                                                     namespace="env")
    k8_dashboard_url: str = k8s.get_service_hostname(
        name="kubernetes-dashboard",
        k8s_context=k8s_context,
        namespace="kubernetes-dashboard")

    context.landing_page_url = f"https://{landing_page_url}"
    if context.cognito_external_provider:
        context.cognito_external_provider_redirect = context.landing_page_url
    context.k8_dashboard_url = f"https://{k8_dashboard_url}"

    _update_elbs(context=context)

    ContextSerDe.dump_context_to_ssm(context=context)
    _logger.debug("Kubectl data fetched successfully.")
    def deploy_env(env_name: str, manifest_dir: str) -> None:
        docker.login(context=context)
        _logger.debug("DockerHub and ECR Logged in")
        cdk_toolkit.deploy(context=context)
        _logger.debug("CDK Toolkit Stack deployed")
        env.deploy(
            context=context,
            eks_system_masters_roles_changes=changeset.eks_system_masters_roles_changeset if changeset else None,
        )

        _logger.debug("Env Stack deployed")
        eksctl.deploy_env(
            context=context,
            changeset=changeset,
        )
        _logger.debug("EKS Environment Stack deployed")
        kubectl.deploy_env(context=context)
        _logger.debug("Kubernetes Environment components deployed")

        helm.deploy_env(context=context)
        _logger.debug("Helm Charts installed")

        k8s_context = utils.get_k8s_context(context=context)
        kubectl.fetch_kubectl_data(context=context, k8s_context=k8s_context)
        ContextSerDe.dump_context_to_ssm(context=context)
        _logger.debug("Updating userpool redirect")
        _update_userpool_client(context=context)
        _update_userpool(context=context)
Esempio n. 4
0
def deploy_toolkit(
    context: "Context",
    username: Optional[str],
    password: Optional[str],
    msg_ctx: MessagesContext,
    top_level: str = "orbit",
) -> None:
    credential_received: bool = username is not None and password is not None
    stack_exist: bool = cfn.does_stack_exist(
        stack_name=context.toolkit.stack_name)
    credential_exist: bool = dockerhub.does_credential_exist(
        context=context) if stack_exist else False
    image_manifests = [
        cast(ImageManifest, getattr(context.images, i))
        for i in context.images.names
    ]
    credential_required: bool = any([
        im.get_source(account_id=context.account_id,
                      region=context.region) == "dockerhub"
        for im in image_manifests
    ])

    if stack_exist:
        if credential_required and not credential_exist and not credential_received:
            username, password = _request_dockerhub_credential(msg_ctx=msg_ctx)
            dockerhub.store_credential(context=context,
                                       username=username,
                                       password=password)
            credential_exist = True
        elif credential_received:
            dockerhub.store_credential(
                context=context,
                username=cast(str, username),
                password=cast(str, password),
            )
            credential_exist = True
    else:
        context.toolkit.deploy_id = "".join(
            random.choice(string.ascii_lowercase) for i in range(6))
        if credential_required and not credential_received:
            username, password = _request_dockerhub_credential(msg_ctx=msg_ctx)
            credential_exist = False

    msg_ctx.progress(6)
    _logger.debug("context.toolkit.deploy_id: %s", context.toolkit.deploy_id)
    template_filename: str = toolkit.synth(context=context,
                                           top_level=top_level)
    cfn.deploy_template(stack_name=context.toolkit.stack_name,
                        filename=template_filename,
                        env_tag=context.env_tag,
                        s3_bucket=None)
    ContextSerDe.fetch_toolkit_data(context=context)
    ContextSerDe.dump_context_to_ssm(context=context)

    if credential_exist is False:
        dockerhub.store_credential(
            context=context,
            username=cast(str, username),
            password=cast(str, password),
        )
def deploy_credentials(filename: str, username: str, password: str, registry: str, debug: bool) -> None:
    with MessagesContext("Deploying", debug=debug) as msg_ctx:
        msg_ctx.progress(2)

        manifest: "Manifest" = ManifestSerDe.load_manifest_from_file(filename=filename, type=Manifest)
        msg_ctx.info(f"Manifest loaded: {filename}")
        msg_ctx.progress(3)

        context_parameter_name: str = f"/orbit/{manifest.name}/context"
        if not ssm.does_parameter_exist(name=context_parameter_name):
            msg_ctx.error(f"Orbit Environment {manifest.name} cannot be found in the current account and region.")
            return

        context: "Context" = ContextSerDe.load_context_from_manifest(manifest=manifest)
        msg_ctx.info("Current Context loaded")
        msg_ctx.progress(4)

        msg_ctx.info("Encrypting credentials with Toolkit KMS Key")
        ciphertext = kms.encrypt(
            context=context, plaintext=json.dumps({registry: {"username": username, "password": password}})
        )
        msg_ctx.progress(20)

        msg_ctx.info("Starting Remote CodeBuild to deploy credentials")

        deploy.deploy_credentials(env_name=context.name, ciphertext=ciphertext)

        msg_ctx.info("Registry Credentials deployed")
        msg_ctx.progress(98)

        if cfn.does_stack_exist(stack_name=context.env_stack_name):
            context = ContextSerDe.load_context_from_ssm(env_name=manifest.name, type=Context)
            msg_ctx.info(f"Context updated: {filename}")
        msg_ctx.progress(100)
Esempio n. 6
0
def fetch_cluster_data(context: "Context", cluster_name: str) -> None:
    _logger.debug("Fetching Cluster data...")
    cluster_data = cast(Dict[str, Any], eks.describe_cluster(cluster_name=cluster_name))

    context.eks_oidc_provider = cluster_data["cluster"]["identity"]["oidc"]["issuer"].replace("https://", "")
    context.cluster_sg_id = cluster_data["cluster"]["resourcesVpcConfig"]["clusterSecurityGroupId"]
    ContextSerDe.dump_context_to_ssm(context=context)
    _logger.debug("Cluster data fetched successfully.")
Esempio n. 7
0
def _team(context: "Context", team_context: "TeamContext",
          output_path: str) -> None:
    input = os.path.join(MODELS_PATH, "teams", "00-team.yaml")
    output = os.path.join(output_path, f"{team_context.name}-00-team.yaml")

    with open(input, "r") as file:
        content: str = file.read()

    content = utils.resolve_parameters(
        content,
        dict(
            team=team_context.name,
            efsid=context.shared_efs_fs_id,
            efsapid=team_context.efs_ap_id,
            efsprivateapid=team_context.efs_private_ap_id
            if team_context.efs_private_ap_id else "",
            account_id=context.account_id,
            env_name=context.name,
            role_prefix=f"/{context.role_prefix}/"
            if context.role_prefix else "/",
            team_kms_key_arn=team_context.team_kms_key_arn,
            team_security_group_id=team_context.team_security_group_id,
            cluster_pod_security_group_id=context.cluster_pod_sg_id,
            team_context=ContextSerDe.dump_context_to_str(team_context),
            env_context=ContextSerDe.dump_context_to_str(context),
            region=context.region,
        ),
    )
    _logger.debug("Kubectl Team %s manifest:\n%s", team_context.name, content)
    with open(output, "w") as file:
        file.write(content)

    # team rbac role
    input = os.path.join(MODELS_PATH, "teams", "01-team-rbac-role.yaml")
    output = os.path.join(output_path,
                          f"{team_context.name}-01-team-rbac-role.yaml")

    with open(input, "r") as file:
        content = file.read()
    content = utils.resolve_parameters(
        content, dict(env_name=context.name, team=team_context.name))
    with open(output, "w") as file:
        file.write(content)

    # bind to admin role
    if team_context.k8_admin:
        # user service account
        input = os.path.join(MODELS_PATH, "teams", "02-admin-binding.yaml")
        output = os.path.join(output_path,
                              f"{team_context.name}-02-admin-binding.yaml")

        with open(input, "r") as file:
            content = file.read()
        content = utils.resolve_parameters(content,
                                           dict(team=team_context.name))
        with open(output, "w") as file:
            file.write(content)
def deploy_env(
    filename: str,
    debug: bool,
) -> None:
    with MessagesContext("Deploying", debug=debug) as msg_ctx:
        msg_ctx.progress(2)

        manifest: "Manifest" = ManifestSerDe.load_manifest_from_file(filename=filename, type=Manifest)
        msg_ctx.info(f"Manifest loaded: {filename}")
        msg_ctx.progress(5)

        manifest_dir: str = os.path.dirname(os.path.abspath(filename))
        _logger.debug("manifest directory is set to %s", manifest_dir)

        manifest_validations(manifest)

        context: "Context" = ContextSerDe.load_context_from_manifest(manifest=manifest)

        msg_ctx.info("Current Context loaded")
        msg_ctx.progress(10)

        _logger.debug("Inspecting possible manifest changes...")
        changeset: "Changeset" = extract_changeset(manifest=manifest, context=context, msg_ctx=msg_ctx)
        _logger.debug(f"Changeset:\n{dump_changeset_to_str(changeset=changeset)}")
        msg_ctx.progress(15)

        _deploy_toolkit(
            context=context,
        )
        msg_ctx.info("Toolkit deployed")
        msg_ctx.progress(30)

        deploy.deploy_env(
            env_name=context.name,
            manifest_dir=manifest_dir,
        )

        msg_ctx.info("Orbit Workbench deployed")
        msg_ctx.progress(98)

        if cfn.does_stack_exist(stack_name=context.env_stack_name):
            context = ContextSerDe.load_context_from_manifest(manifest=manifest)
            msg_ctx.info(f"Context updated: {filename}")
        msg_ctx.progress(99)

        if context.cognito_users_url:
            msg_ctx.tip(f"Add users: {stylize(context.cognito_users_url, underline=True)}")
        else:
            RuntimeError("Cognito Users URL not found.")
        if context.landing_page_url:
            msg_ctx.tip(f"Access Orbit Workbench: {stylize(f'{context.landing_page_url}/orbit/login', underline=True)}")
        else:
            RuntimeError("Landing Page URL not found.")
        msg_ctx.progress(100)
def deploy_teams(
    filename: str,
    debug: bool,
) -> None:
    with MessagesContext("Deploying", debug=debug) as msg_ctx:
        msg_ctx.progress(2)

        manifest: "Manifest" = ManifestSerDe.load_manifest_from_file(filename=filename, type=Manifest)
        msg_ctx.info(f"Manifest loaded: {filename}")
        msg_ctx.info(f"Teams: {','.join([t.name for t in manifest.teams])}")
        msg_ctx.progress(5)

        manifest_dir: str = os.path.dirname(os.path.abspath(filename))
        _logger.debug("manifest directory is set to %s", manifest_dir)

        context_parameter_name: str = f"/orbit/{manifest.name}/context"
        if not ssm.does_parameter_exist(name=context_parameter_name):
            msg_ctx.error(f"Orbit Environment {manifest.name} cannot be found in the current account and region.")
            return

        context: "Context" = ContextSerDe.load_context_from_manifest(manifest=manifest)
        msg_ctx.info("Current Context loaded")
        msg_ctx.info(f"Teams: {','.join([t.name for t in context.teams])}")
        msg_ctx.progress(10)

        _logger.debug("Inspecting possible manifest changes...")
        changeset: "Changeset" = extract_changeset(manifest=manifest, context=context, msg_ctx=msg_ctx)
        _logger.debug(f"Changeset:\n{dump_changeset_to_str(changeset=changeset)}")

        msg_ctx.progress(30)

        deploy.deploy_teams(
            env_name=context.name,
            manifest_dir=manifest_dir,
        )

        msg_ctx.info("Orbit Workbench deployed")
        msg_ctx.progress(98)

        if cfn.does_stack_exist(stack_name=context.env_stack_name):
            context = ContextSerDe.load_context_from_ssm(env_name=manifest.name, type=Context)
            msg_ctx.info(f"Context updated: {filename}")
        msg_ctx.progress(99)

        if context.user_pool_id:
            cognito_users_url = orbit_cognito.get_users_url(user_pool_id=context.user_pool_id, region=context.region)
            msg_ctx.tip(f"Add users: {stylize(cognito_users_url, underline=True)}")

        if context.landing_page_url:
            msg_ctx.tip(f"Access Orbit Workbench: {stylize(f'{context.landing_page_url}/orbit/login', underline=True)}")
        else:
            raise RuntimeError("Landing Page URL not found.")
        msg_ctx.progress(100)
def _deploy_toolkit(
    context: "Context",
    top_level: str = "orbit",
) -> None:
    stack_exist: bool = cfn.does_stack_exist(stack_name=context.toolkit.stack_name)
    if not stack_exist:
        context.toolkit.deploy_id = "".join(random.choice(string.ascii_lowercase) for i in range(6))
    _logger.debug("context.toolkit.deploy_id: %s", context.toolkit.deploy_id)
    template_filename: str = toolkit.synth(context=context, top_level=top_level)
    cfn.deploy_template(
        stack_name=context.toolkit.stack_name, filename=template_filename, env_tag=context.env_tag, s3_bucket=None
    )
    ContextSerDe.fetch_toolkit_data(context=context)
    ContextSerDe.dump_context_to_ssm(context=context)
Esempio n. 11
0
def _deploy_images_batch(context: "Context", images: List[Tuple[str, Optional[str], Optional[str], List[str]]]) -> None:
    _logger.debug("images:\n%s", images)

    new_images_manifest = {name: getattr(context.images, name) for name in context.images.names}
    max_workers = 5
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
        futures: List[Future[Any]] = []
        name: str = ""
        dir: Optional[str] = None
        script: Optional[str] = None
        build_args: List[str] = []

        for name, dir, script, build_args in images:
            _logger.debug("name: %s | script: %s", name, script)

            path = os.path.join(os.getcwd(), "bundle", name)
            _logger.debug("path: %s", path)

            image_attr_name = name.replace("-", "_")
            image_def: ImageManifest = getattr(context.images, image_attr_name)
            tag = image_def.version
            if image_def.get_source(account_id=context.account_id, region=context.region) == "code":
                dirs: List[Tuple[str, str]] = [(path, cast(str, dir))]
            else:
                dirs = []

            bundle_path = bundle.generate_bundle(command_name=f"deploy_image-{name}", context=context, dirs=dirs)
            _logger.debug("bundle_path: %s", bundle_path)
            script_str = "NO_SCRIPT" if script is None else script
            build_args = [] if build_args is None else build_args
            buildspec = codebuild.generate_spec(
                context=context,
                plugins=False,
                cmds_build=[
                    "orbit remote --command _deploy_image "
                    f"{context.name} {name} {dir} {script_str} {' '.join(build_args)}"
                ],
            )
            new_images_manifest[image_attr_name] = ImageManifest(
                repository=f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/orbit-{context.name}/{name}",
                version=tag,
                path=None,
            )
            futures.append(executor.submit(_deploy_image_remotely, context, name, bundle_path, buildspec))

        for f in futures:
            f.result()

        context.images = ImagesManifest(**new_images_manifest)
        ContextSerDe.dump_context_to_ssm(context=context)
Esempio n. 12
0
def deploy_team(context: "Context", manifest: Manifest,
                team_manifest: TeamManifest) -> None:
    # Pull team spacific custom cfn plugin, trigger pre_hook
    team_context: Optional["TeamContext"] = create_team_context_from_manifest(
        manifest=manifest, team_manifest=team_manifest)
    _logger.debug(f"team_context={team_context}")
    if team_context:
        _logger.debug(f"team_context.plugins={team_context.plugins}")
        _logger.debug("Calling team pre_hook")
        for plugin in team_context.plugins:
            hook: plugins.HOOK_TYPE = plugins.PLUGINS_REGISTRIES.get_hook(
                context=context,
                team_name=team_context.name,
                plugin_name=plugin.plugin_id,
                hook_name="pre_hook",
            )
            if hook is not None:
                _logger.debug(f"Found pre_hook for plugin_id {plugin}")
                hook(plugin.plugin_id, context, team_context,
                     plugin.parameters)
        _logger.debug("End of pre_hook plugin execution")
    else:
        _logger.debug(
            f"Skipping pre_hook for unknown Team: {team_manifest.name}")

    args = [context.name, team_manifest.name]
    cdk.deploy(
        context=context,
        stack_name=f"orbit-{manifest.name}-{team_manifest.name}",
        app_filename=os.path.join(ORBIT_CLI_ROOT, "remote_files", "cdk",
                                  "team.py"),
        args=args,
    )
    team_context = context.get_team_by_name(name=team_manifest.name)
    if team_context:
        team_context.fetch_team_data()
    else:
        team_context = create_team_context_from_manifest(
            manifest=manifest, team_manifest=team_manifest)
        team_context.fetch_team_data()
        context.teams.append(team_context)

    _logger.debug(
        f"team_context.helm_repository: s3://{context.toolkit.s3_bucket}/helm/repositories/teams/{team_context.name}"
    )
    team_context.team_helm_repository = f"s3://{context.toolkit.s3_bucket}/helm/repositories/teams/{team_context.name}"
    team_context.user_helm_repository = f"s3://{context.toolkit.s3_bucket}/helm/repositories/user/{team_context.name}"
    ContextSerDe.dump_context_to_ssm(context=context)
def cdk_prep_team_handler(stack_class: Type["Stack"]) -> None:
    _logger.debug("sys.argv: %s", sys.argv)
    if len(sys.argv) != 5:
        raise ValueError(f"Unexpected number of values in sys.argv ({len(sys.argv)}) - {sys.argv}.")

    stack_name: str = sys.argv[1]
    # team_name: str = sys.argv[3]
    parameters: Dict[str, Any] = _deserialize_parameters(parameters=sys.argv[4])
    context: "Context" = ContextSerDe.load_context_from_ssm(env_name=sys.argv[2], type=Context)

    # Can not find /orbit/env_name/teams ssm param.
    # team_context = context.get_team_by_name(name=team_name)
    # if team_context is None:
    #     raise ValueError(f"Team {team_name} not found in the context.")

    outdir = os.path.join(
        ".orbit.out",
        context.name,
        "cdk",
        stack_name,
    )
    shutil.rmtree(outdir, ignore_errors=True)
    os.makedirs(outdir, exist_ok=True)

    # Can't be imported globally because we only have CDK installed on CodeBuild
    from aws_cdk.core import App

    app = App(outdir=outdir)
    stack_class(app, stack_name, context, parameters)  # type: ignore
    app.synth(force=True)
def deploy_credentials(env_name: str, ciphertext: str) -> None:
    context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=Context)
    _logger.debug("Context loaded.")

    @codeseeder.remote_function("orbit", codebuild_role=context.toolkit.admin_role)
    def deploy_credentials(env_name: str, ciphertext: str) -> None:
        new_credentials = json.loads(kms.decrypt(context=context, ciphertext=ciphertext))
        secret_id = f"orbit-{env_name}-docker-credentials"
        existing_credentials = secretsmanager.get_secret_value(secret_id=secret_id)
        for registry, creds in new_credentials.items():
            username = creds.get("username", "")
            password = creds.get("password", "")
            try:
                subprocess.check_call(
                    f"docker login --username '{username}' --password '{password}' {registry}", shell=True
                )
            except Exception as e:
                _logger.error("Invalid Registry Credentials")
                _logger.exception(e)
                return
            else:
                existing_credentials = {**existing_credentials, **new_credentials}
        secretsmanager.put_secret_value(secret_id=secret_id, secret=existing_credentials)
        _logger.debug("Registry Credentials deployed")

    deploy_credentials(env_name=env_name, ciphertext=ciphertext)
Esempio n. 15
0
def destroy_teams(args: Tuple[str, ...]) -> None:
    _logger.debug("args %s", args)
    env_name: str = args[0]
    context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env_name,
                                                            type=Context)
    _logger.debug("context.name %s", context.name)

    plugins.PLUGINS_REGISTRIES.load_plugins(context=context,
                                            plugin_changesets=[],
                                            teams_changeset=None)
    kubectl.write_kubeconfig(context=context)
    _logger.debug("Plugins loaded")
    for team_context in context.teams:
        plugins.PLUGINS_REGISTRIES.destroy_team_plugins(
            context=context, team_context=team_context)
    _logger.debug("Plugins destroyed")
    for team_context in context.teams:
        helm.destroy_team(context=context, team_context=team_context)
    _logger.debug("Helm Charts uninstalled")
    kubectl.destroy_teams(context=context)
    _logger.debug("Kubernetes Team components destroyed")
    eksctl.destroy_teams(context=context)
    _logger.debug("EKS Team Stacks destroyed")
    teams.destroy_all(context=context)
    _logger.debug("Teams Stacks destroyed")
    ssm.cleanup_teams(env_name=context.name)
def destroy_toolkit(
    env: str,
    debug: bool,
) -> None:
    with MessagesContext("Destroying Environment Toolkit",
                         debug=debug) as msg_ctx:
        msg_ctx.progress(10)

        context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env,
                                                                type=Context)
        msg_ctx.info("Context loaded")
        msg_ctx.progress(25)

        try:
            _destroy_toolkit(env_name=context.name)
        except botocore.exceptions.ClientError as ex:
            error = ex.response["Error"]
            if "does not exist" not in error["Message"]:
                raise
            _logger.debug(f"Skipping toolkit destroy: {error['Message']}")
        msg_ctx.info("Toolkit destroyed")
        ssm.cleanup_env(env_name=context.name)

        msg_ctx.info("Toolkit destroyed")
        msg_ctx.progress(100)
Esempio n. 17
0
def build_image(
    env: str,
    dir: Optional[str],
    name: str,
    script: Optional[str],
    build_args: Optional[List[str]],
    timeout: int = 30,
    debug: bool = False,
    source_registry: Optional[str] = None,
    source_repository: Optional[str] = None,
    source_version: Optional[str] = None,
) -> None:
    with MessagesContext("Deploying Docker Image", debug=debug) as msg_ctx:
        context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env, type=Context)
        msg_ctx.info("Manifest loaded")
        if cfn.does_stack_exist(stack_name=f"orbit-{context.name}") is False:
            msg_ctx.error("Please, deploy your environment before deploying any additional docker image")
            return
        msg_ctx.progress(3)
        if dir:
            dirs = [(dir, name)]
        else:
            dirs = []
        bundle_path = bundle.generate_bundle(command_name=f"deploy_image-{name}", context=context, dirs=dirs)
        msg_ctx.progress(5)

        script_str = "NO_SCRIPT" if script is None else script
        source_str = "NO_REPO" if source_registry is None else f"{source_registry} {source_repository} {source_version}"
        build_args = [] if build_args is None else build_args
        buildspec = codebuild.generate_spec(
            context=context,
            plugins=False,
            cmds_build=[
                f"orbit remote --command build_image " f"{env} {name} {script_str} {source_str} {' '.join(build_args)}"
            ],
            changeset=None,
        )
        msg_ctx.progress(6)

        remote.run(
            command_name=f"deploy_image-{name}",
            context=context,
            bundle_path=bundle_path,
            buildspec=buildspec,
            codebuild_log_callback=msg_ctx.progress_bar_callback,
            timeout=timeout,
        )
        msg_ctx.info("Docker Image deploy into ECR")

        address = (
            f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/orbit-{context.name}/{name}"
            if name in [n.replace("_", "-") for n in context.images.names]
            else f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/orbit-{context.name}/users/{name}"
        )

        msg_ctx.info(f"ECR Image Address={address}")
        msg_ctx.tip(f"ECR Image Address: {stylize(address, underline=True)}")
        msg_ctx.progress(100)
Esempio n. 18
0
def destroy_env(env: str, debug: bool) -> None:
    with MessagesContext("Destroying", debug=debug) as msg_ctx:
        ssm.cleanup_changeset(env_name=env)
        ssm.cleanup_manifest(env_name=env)

        if ssm.does_parameter_exist(name=f"/orbit/{env}/context") is False:
            msg_ctx.info(f"Environment {env} not found. Destroying only possible remaining resources.")
            elb.delete_load_balancers(env_name=env)
            destroy_remaining_resources(env_name=env, top_level="orbit")
            msg_ctx.progress(100)
            return

        context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env, type=Context)
        msg_ctx.info("Context loaded")
        msg_ctx.info(f"Teams: {','.join([t.name for t in context.teams])}")
        msg_ctx.progress(2)

        if any(cfn.does_stack_exist(stack_name=t.stack_name) for t in context.teams):
            msg_ctx.error("Found Teams dependent on the Envrionment.")
            return

        if (
            cfn.does_stack_exist(stack_name=context.env_stack_name)
            or cfn.does_stack_exist(stack_name=context.toolkit.stack_name)
            or cfn.does_stack_exist(stack_name=context.cdk_toolkit.stack_name)
        ):
            bundle_path = bundle.generate_bundle(command_name="destroy", context=context)
            msg_ctx.progress(5)

            buildspec = codebuild.generate_spec(
                context=context,
                plugins=True,
                cmds_build=[f"orbit remote --command destroy_env {env}"],
                changeset=None,
            )
            remote.run(
                command_name="destroy",
                context=context,
                bundle_path=bundle_path,
                buildspec=buildspec,
                codebuild_log_callback=msg_ctx.progress_bar_callback,
                timeout=45,
            )

        msg_ctx.info("Env destroyed")
        msg_ctx.progress(95)

        try:
            destroy_toolkit(env_name=context.name)
        except botocore.exceptions.ClientError as ex:
            error = ex.response["Error"]
            if "does not exist" not in error["Message"]:
                raise
            _logger.debug(f"Skipping toolkit destroy: {error['Message']}")
        msg_ctx.info("Toolkit destroyed")
        ssm.cleanup_env(env_name=context.name)

        msg_ctx.progress(100)
Esempio n. 19
0
def list_env(env: str, variable: str) -> None:
    ssm = utils.boto3_client("ssm")
    res = ssm.get_parameters_by_path(Path="/orbit", Recursive=True)
    env_info: Dict[str, str] = {}
    while True:
        params = res["Parameters"]
        for p in params:
            if not p["Name"].endswith("context") or "teams" in p["Name"]:
                continue
            if len(env) > 0 and p["Name"].startswith(f"//orbit/{env}"):
                continue
            env_name = p["Name"].split("/")[2]
            context: "Context" = ContextSerDe.load_context_from_ssm(
                env_name=env_name, type=Context)
            _logger.debug(f"found env: {env_name}")
            if context.k8_dashboard_url:
                k8_dashboard_url = context.k8_dashboard_url
            else:
                k8_dashboard_url = ""
            if len(context.teams) > 0:
                teams_list: str = ",".join([x.name for x in context.teams])
            else:
                teams_list = ""

            if variable == "landing-page":
                print(context.landing_page_url)
            elif variable == "toolkitbucket":
                print(context.toolkit.s3_bucket)
            elif variable == "teams":
                print(f"[{teams_list}]")
            elif variable == "all":
                env_info[env_name] = (
                    f"LandingPage={context.landing_page_url}, "
                    f"Teams=[{teams_list}], "
                    f"ToolkitBucket={context.toolkit.s3_bucket}"
                    f"K8Dashboard={k8_dashboard_url}")
            else:
                raise Exception(f"Unknown --variable option {variable}")

        if "NextToken" in res:
            res = ssm.get_parameters_by_path(Path="/orbit",
                                             Recursive=True,
                                             NextToken=res["NextToken"])
        else:
            break

    if variable == "all":
        if len(env_info) == 0:
            click.echo("There are no Orbit environments available")
            return
        else:
            print_list(
                tittle="Available Orbit environments:",
                items=[
                    f"Name={k}{stylize(',')}{v}" for k, v in env_info.items()
                ],
            )
Esempio n. 20
0
def fetch_kubectl_data(context: "Context", k8s_context: str) -> None:
    _logger.debug("Fetching Kubectl data...")

    ingress_url: str = k8s.get_ingress_dns(name="istio-ingress",
                                           k8s_context=k8s_context,
                                           namespace="istio-system")

    if context.networking.frontend.custom_domain_name:
        context.landing_page_url = f"https://{context.networking.frontend.custom_domain_name}"
    else:
        context.landing_page_url = f"https://{ingress_url}"

    if context.cognito_external_provider:
        context.cognito_external_provider_redirect = context.landing_page_url

    _update_elbs(context=context)

    ContextSerDe.dump_context_to_ssm(context=context)
    _logger.debug("Kubectl data fetched successfully.")
Esempio n. 21
0
def destroy_foundation(args: Tuple[str, ...]) -> None:
    _logger.debug("args %s", args)
    env_name: str = args[0]
    context: "FoundationContext" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=FoundationContext)
    _logger.debug("context.name %s", context.name)

    foundation.destroy(context=context)
    _logger.debug("Demo Stack destroyed")
    cdk_toolkit.destroy(context=context)
    _logger.debug("CDK Toolkit Stack destroyed")
Esempio n. 22
0
def list_env(env: str, variable: str) -> None:
    ssm = utils.boto3_client("ssm")
    res = ssm.get_parameters_by_path(Path="/orbit", Recursive=True)
    env_info: Dict[str, Any] = {}
    if env and len(env) > 0:
        _logger.debug(f"looking for {env}")
    while True:
        params = res["Parameters"]
        for p in params:
            if not p["Name"].endswith("context") or "teams" in p["Name"]:
                continue
            env_name = p["Name"].split("/")[2]
            if len(env) > 0 and not env_name == env:
                continue
            env_name = p["Name"].split("/")[2]
            context: "Context" = ContextSerDe.load_context_from_ssm(
                env_name=env_name, type=Context)
            _logger.debug(f"found env: {env_name}")
            if context.k8_dashboard_url:
                k8_dashboard_url = context.k8_dashboard_url
            else:
                k8_dashboard_url = ""
            if len(context.teams) > 0:
                teams_list: List[str] = [x.name for x in context.teams]
            else:
                teams_list = []

            if variable == "landing-page":
                print(context.landing_page_url)
            elif variable == "toolkitbucket":
                print(context.toolkit.s3_bucket)
            elif variable == "all":
                env_info[env_name] = {
                    "LandingPage": context.landing_page_url,
                    "Teams": teams_list,
                    "ToolkitBucket": context.toolkit.s3_bucket,
                    "K8Dashboard": k8_dashboard_url,
                }
            else:
                raise Exception(f"Unknown --variable option {variable}")

        if "NextToken" in res:
            res = ssm.get_parameters_by_path(Path="/orbit",
                                             Recursive=True,
                                             NextToken=res["NextToken"])
        else:
            break

    if variable == "all":
        if len(env_info) == 0:
            click.echo("There are no Orbit environments available")
            return
        else:
            print("Available Orbit environments:")
            print(json.dumps(env_info, indent=4, sort_keys=True))
def destroy_images(env: str) -> None:
    context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env,
                                                            type=Context)
    _logger.debug("env %s", env)

    @codeseeder.remote_function("orbit",
                                codebuild_role=context.toolkit.admin_role)
    def destroy_images(env: str) -> None:
        ecr.cleanup_remaining_repos(env_name=env)

    destroy_images(env=env)
Esempio n. 24
0
def _deploy_image(
    env: str,
    dir: str,
    name: str,
    script: Optional[str],
    build_args: Optional[List[str]],
    region: Optional[str],
    debug: bool,
) -> None:
    with MessagesContext("Deploying Docker Image", debug=debug) as msg_ctx:
        context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env,
                                                                type=Context)

        if cfn.does_stack_exist(stack_name=f"orbit-{context.name}") is False:
            msg_ctx.error(
                "Please, deploy your environment before deploy any addicional docker image"
            )
            return

        plugins.PLUGINS_REGISTRIES.load_plugins(
            context=context,
            msg_ctx=msg_ctx,
            plugin_changesets=[],
            teams_changeset=None,
        )
        msg_ctx.progress(3)

        bundle_path = bundle.generate_bundle(
            command_name=f"deploy_image-{name}",
            context=context,
            dirs=[(dir, name)])
        msg_ctx.progress(4)
        script_str = "NO_SCRIPT" if script is None else script
        build_args = [] if build_args is None else build_args
        buildspec = codebuild.generate_spec(
            context=context,
            plugins=True,
            cmds_build=[
                f"orbit remote --command _deploy_image {env} {name} {dir} {script_str} {' '.join(build_args)}"
            ],
            changeset=None,
        )
        remote.run(
            command_name=f"deploy_image-{name}",
            context=context,
            bundle_path=bundle_path,
            buildspec=buildspec,
            codebuild_log_callback=msg_ctx.progress_bar_callback,
            timeout=30,
        )
        msg_ctx.info("Docker Image deploy into ECR")
        address = f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/orbit-{context.name}-{name}"
        msg_ctx.tip(f"ECR Image Address: {stylize(address, underline=True)}")
        msg_ctx.progress(100)
Esempio n. 25
0
def destroy_foundation(env: str, debug: bool) -> None:
    with MessagesContext("Destroying", debug=debug) as msg_ctx:
        ssm.cleanup_changeset(env_name=env, top_level="orbit-foundation")
        ssm.cleanup_manifest(env_name=env, top_level="orbit-foundation")

        if ssm.does_parameter_exist(name=f"/orbit-foundation/{env}/context") is False:
            msg_ctx.info(f"Foundation {env} not found. Destroying only possible remaining resources.")
            destroy_remaining_resources(env_name=env, top_level="orbit-foundation")
            msg_ctx.progress(100)
            return

        context: "FoundationContext" = ContextSerDe.load_context_from_ssm(env_name=env, type=FoundationContext)
        msg_ctx.info("Context loaded")
        msg_ctx.progress(2)

        msg_ctx.progress(4)

        if (
            cfn.does_stack_exist(stack_name=cast(str, context.stack_name))
            or cfn.does_stack_exist(stack_name=context.toolkit.stack_name)
            or cfn.does_stack_exist(stack_name=context.cdk_toolkit.stack_name)
        ):
            bundle_path = bundle.generate_bundle(command_name="destroy", context=cast(Context, context))
            msg_ctx.progress(5)

            buildspec = codebuild.generate_spec(
                context=cast(Context, context),
                plugins=False,
                cmds_build=[f"orbit remote --command destroy_foundation {env}"],
                changeset=None,
            )
            remote.run(
                command_name="destroy",
                context=cast(Context, context),
                bundle_path=bundle_path,
                buildspec=buildspec,
                codebuild_log_callback=msg_ctx.progress_bar_callback,
                timeout=45,
            )
        msg_ctx.info("Foundation destroyed")
        msg_ctx.progress(95)

        try:
            destroy_toolkit(env_name=context.name, top_level="orbit-foundation")
        except botocore.exceptions.ClientError as ex:
            error = ex.response["Error"]
            if "does not exist" not in error["Message"]:
                raise
            _logger.debug(f"Skipping toolkit destroy: {error['Message']}")
        msg_ctx.info("Toolkit destroyed")
        ssm.cleanup_env(env_name=context.name, top_level="orbit-foundation")

        msg_ctx.progress(100)
Esempio n. 26
0
def main() -> None:
    _logger.debug("sys.argv: %s", sys.argv)
    if len(sys.argv) == 3:
        context: "Context" = ContextSerDe.load_context_from_ssm(
            env_name=sys.argv[1], type=Context)
        team_name: str = sys.argv[2]
    else:
        raise ValueError("Unexpected number of values in sys.argv.")

    changeset: Optional["Changeset"] = load_changeset_from_ssm(
        env_name=context.name)
    _logger.debug("Changeset loaded.")

    team_policies: Optional[List[str]] = None
    image: Optional[str] = None

    if changeset and changeset.teams_changeset and team_name in changeset.teams_changeset.added_teams_names:
        manifest: Optional["Manifest"] = ManifestSerDe.load_manifest_from_ssm(
            env_name=sys.argv[1], type=Manifest)
        if manifest is None:
            raise ValueError("manifest is None!")
        team_manifest: Optional["TeamManifest"] = manifest.get_team_by_name(
            name=team_name)
        if team_manifest:
            team_policies = team_manifest.policies
            image = team_manifest.image
        else:
            raise ValueError(f"{team_name} not found in manifest!")
    else:
        team_context: Optional["TeamContext"] = context.get_team_by_name(
            name=team_name)
        if team_context:
            team_policies = team_context.policies
            image = team_context.image
        else:
            raise ValueError(f"Team {team_name} not found in the context.")

    if team_policies is None:
        raise ValueError("team_policies is None!")

    stack_name: str = f"orbit-{context.name}-{team_name}"
    outdir = os.path.join(".orbit.out", context.name, "cdk", stack_name)
    os.makedirs(outdir, exist_ok=True)
    shutil.rmtree(outdir)
    app = App(outdir=outdir)
    Team(scope=app,
         id=stack_name,
         context=context,
         team_name=team_name,
         team_policies=team_policies,
         image=image)
    app.synth(force=True)
Esempio n. 27
0
def deploy_foundation(args: Tuple[str, ...]) -> None:
    _logger.debug("args: %s", args)
    if len(args) != 1:
        raise ValueError("Unexpected number of values in args")
    env_name: str = args[0]
    context: "FoundationContext" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=FoundationContext)
    _logger.debug("Context loaded.")
    docker.login(context=context)
    _logger.debug("DockerHub and ECR Logged in")
    cdk_toolkit.deploy(context=context)
    _logger.debug("CDK Toolkit Stack deployed")
    foundation.deploy(context=context)
    _logger.debug("Demo Stack deployed")
Esempio n. 28
0
def list_images(env: str, region: Optional[str]) -> None:
    context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env,
                                                            type=Context)
    names = utils.extract_images_names(env_name=env)
    if names:
        print_list(
            tittle=
            f"Available docker images into the {stylize(context.name)} env:",
            items=[k for k in names],
        )
    else:
        click.echo(
            f"There is no docker image(s) into the {stylize(context.name)} env."
        )
Esempio n. 29
0
def destroy_env(args: Tuple[str, ...]) -> None:
    _logger.debug("args %s", args)
    env_name: str = args[0]
    context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=Context)
    _logger.debug("context.name %s", context.name)

    kubectl.destroy_env(context=context)
    _logger.debug("Kubernetes Environment components destroyed")
    eksctl.destroy_env(context=context)
    _logger.debug("EKS Environment Stacks destroyed")
    env.destroy(context=context)
    _logger.debug("Env Stack destroyed")
    cdk_toolkit.destroy(context=context)
    _logger.debug("CDK Toolkit Stack destroyed")
Esempio n. 30
0
def delete_image(args: Tuple[str, ...]) -> None:
    _logger.debug("args %s", args)
    env_name: str = args[0]
    context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=Context)

    if len(args) == 2:
        image_name: str = args[1]
    else:
        raise ValueError("Unexpected number of values in args.")

    env.deploy(context=context, add_images=[], remove_images=[image_name], eks_system_masters_roles_changes=None)
    _logger.debug("Env changes deployed")
    ecr.delete_repo(repo=f"orbit-{context.name}-{image_name}")
    _logger.debug("Docker Image Destroyed from ECR")