Beispiel #1
0
def logs(context, timestamps, follow, tail, previous, labels, namespace=None):
  kctl = Kubectl(context, namespace=namespace)

  opts = ''
  if timestamps:
    opts += ' --timestamps'
  if previous:
    opts += ' --previous'
  if follow or config.follow_logs:
    opts += ' --follow'
  if tail or config.tail_logs:
    num_tail = tail if tail else config.tail_logs
    opts += " --tail=%s" % num_tail

  selectors = ["app=%s" % config.project_name, "layer=application"]
  for l in labels:
    if '=' not in l:
      raise HokusaiError("Error: label selectors of the form 'key=value'")
    selectors.append(l)

  pods = kctl.get_objects('pod', selector=(',').join(selectors))
  pods = [pod for pod in pods if pod['status']['phase'] == 'Running']
  containers = []

  for pod in pods:
    for container in pod['spec']['containers']:
      containers.append({'pod': pod['metadata']['name'], 'name': container['name']})

  commands = [kctl.command("logs %s %s%s" % (container['pod'], container['name'], opts)) for container in containers]
  shout_concurrent(commands, print_output=True)
Beispiel #2
0
    def refresh(self):
        deployment_timestamp = datetime.datetime.utcnow().strftime("%s%f")
        for deployment in self.cache:
            patch = {
                "spec": {
                    "template": {
                        "metadata": {
                            "labels": {
                                "deploymentTimestamp": deployment_timestamp
                            }
                        }
                    }
                }
            }
            print_green("Refreshing %s..." % deployment['metadata']['name'])
            shout(
                self.kctl.command(
                    "patch deployment %s -p '%s'" %
                    (deployment['metadata']['name'], json.dumps(patch))))

        print_green("Waiting for refresh to complete...")

        rollout_commands = [
            self.kctl.command("rollout status deployment/%s" %
                              deployment['metadata']['name'])
            for deployment in self.cache
        ]
        return_code = shout_concurrent(rollout_commands)
        if return_code:
            raise HokusaiError("Refresh failed!", return_code=return_code)
Beispiel #3
0
def logs(context, timestamps, follow, tail):
    kctl = Kubectl(context)

    opts = ''
    if timestamps:
        opts += ' --timestamps'
    if follow:
        opts += ' --follow'
    if tail:
        opts += " --tail=%s" % tail

    pods = kctl.get_object('pod',
                           selector="app=%s,layer=application" %
                           config.project_name)
    pods = filter(lambda pod: pod['status']['phase'] == 'Running', pods)
    containers = []

    for pod in pods:
        for container in pod['spec']['containers']:
            containers.append({
                'pod': pod['metadata']['name'],
                'name': container['name']
            })

    commands = [
        kctl.command("logs %s %s%s" %
                     (container['pod'], container['name'], opts))
        for container in containers
    ]
    return shout_concurrent(commands, print_output=True)
Beispiel #4
0
    def refresh(self):
        for deployment in self.cache:
            print_green("Refreshing %s..." % deployment['metadata']['name'],
                        newline_after=True)
            shout(
                self.kctl.command("rollout restart deployment/%s" %
                                  deployment['metadata']['name']))

        print_green("Waiting for refresh to complete...")

        rollout_commands = [
            self.kctl.command("rollout status deployment/%s" %
                              deployment['metadata']['name'])
            for deployment in self.cache
        ]
        return_codes = shout_concurrent(rollout_commands, print_output=True)
        if any(return_codes):
            raise HokusaiError("Refresh failed!")
Beispiel #5
0
    def update(self, tag, constraint):
        print_green("Deploying %s to %s..." % (tag, self.context))

        if self.context != tag:
            ecr = ECR()

            ecr.retag(tag, self.context)
            print_green("Updated tag %s -> %s" % (tag, self.context))

            deployment_tag = "%s--%s" % (
                self.context,
                datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
            ecr.retag(tag, deployment_tag)
            print_green("Updated tag %s -> %s" % (tag, deployment_tag))

        if config.pre_deploy is not None:
            print_green("Running pre-deploy hook '%s' on %s..." %
                        (config.pre_deploy, self.context))
            return_code = CommandRunner(self.context).run(
                tag, config.pre_deploy, constraint=constraint)
            if return_code:
                raise HokusaiError(
                    "Pre-deploy hook failed with return code %s" % return_code,
                    return_code=return_code)

        deployment_timestamp = datetime.datetime.utcnow().strftime("%s%f")
        for deployment in self.cache:
            containers = deployment['spec']['template']['spec']['containers']
            container_names = [container['name'] for container in containers]
            deployment_targets = [{
                "name":
                name,
                "image":
                "%s:%s" % (config.aws_ecr_registry, tag)
            } for name in container_names]
            patch = {
                "spec": {
                    "template": {
                        "metadata": {
                            "labels": {
                                "deploymentTimestamp": deployment_timestamp
                            }
                        },
                        "spec": {
                            "containers": deployment_targets
                        }
                    }
                }
            }
            print_green("Patching deployment %s..." %
                        deployment['metadata']['name'])
            shout(
                self.kctl.command(
                    "patch deployment %s -p '%s'" %
                    (deployment['metadata']['name'], json.dumps(patch))))

        print_green("Waiting for rollout to complete...")

        rollout_commands = [
            self.kctl.command("rollout status deployment/%s" %
                              deployment['metadata']['name'])
            for deployment in self.cache
        ]
        return_code = shout_concurrent(rollout_commands)
        if return_code:
            raise HokusaiError("Deployment failed!", return_code=return_code)

        if config.post_deploy is not None:
            print_green("Running post-deploy hook '%s' on %s..." %
                        (config.post_deploy, self.context))
            return_code = CommandRunner(self.context).run(
                tag, config.post_deploy, constraint=constraint)
            if return_code:
                raise HokusaiError(
                    "Post-deploy hook failed with return code %s" %
                    return_code,
                    return_code=return_code)
Beispiel #6
0
    def update(self,
               tag,
               constraint,
               git_remote,
               timeout,
               resolve_tag_sha1=True):
        if not self.ecr.project_repo_exists():
            raise HokusaiError("Project repo does not exist.  Aborting.")

        if resolve_tag_sha1:
            tag = self.ecr.find_git_sha1_image_tag(tag)
            if tag is None:
                raise HokusaiError(
                    "Could not find a git SHA1 for tag %s.  Aborting." % tag)

        if self.namespace is None:
            print_green("Deploying %s to %s..." % (tag, self.context),
                        newline_after=True)
        else:
            print_green("Deploying %s to %s/%s..." %
                        (tag, self.context, self.namespace),
                        newline_after=True)

        if config.pre_deploy is not None:
            print_green("Running pre-deploy hook '%s'..." % config.pre_deploy,
                        newline_after=True)
            return_code = CommandRunner(self.context,
                                        namespace=self.namespace).run(
                                            tag,
                                            config.pre_deploy,
                                            constraint=constraint,
                                            tty=False)
            if return_code:
                raise HokusaiError(
                    "Pre-deploy hook failed with return code %s" % return_code,
                    return_code=return_code)

        deployment_timestamp = datetime.datetime.utcnow().strftime("%s%f")
        for deployment in self.cache:
            containers = [(container['name'], container['image'])
                          for container in deployment['spec']['template']
                          ['spec']['containers']]
            deployment_targets = [{
                "name":
                name,
                "image":
                "%s:%s" % (self.ecr.project_repo, tag)
            } for name, image in containers if self.ecr.project_repo in image]
            patch = {
                "spec": {
                    "template": {
                        "metadata": {
                            "labels": {
                                "deploymentTimestamp": deployment_timestamp
                            }
                        },
                        "spec": {
                            "containers": deployment_targets
                        }
                    },
                    "progressDeadlineSeconds": timeout
                }
            }

            print_green("Patching deployment %s..." %
                        deployment['metadata']['name'],
                        newline_after=True)
            shout(
                self.kctl.command(
                    "patch deployment %s -p '%s'" %
                    (deployment['metadata']['name'], json.dumps(patch))))

        print_green("Waiting for deployment rollouts to complete...")

        rollout_commands = [
            self.kctl.command("rollout status deployment/%s" %
                              deployment['metadata']['name'])
            for deployment in self.cache
        ]
        return_codes = shout_concurrent(rollout_commands, print_output=True)
        if any(return_codes):
            print_red(
                "One or more deployment rollouts timed out!  Rolling back...",
                newline_before=True,
                newline_after=True)
            rollback_commands = [
                self.kctl.command("rollout undo deployment/%s" %
                                  deployment['metadata']['name'])
                for deployment in self.cache
            ]
            shout_concurrent(rollback_commands, print_output=True)
            raise HokusaiError("Deployment failed!")

        post_deploy_success = True

        if config.post_deploy is not None:
            print_green("Running post-deploy hook '%s'..." %
                        config.post_deploy,
                        newline_after=True)
            return_code = CommandRunner(self.context,
                                        namespace=self.namespace).run(
                                            tag,
                                            config.post_deploy,
                                            constraint=constraint,
                                            tty=False)
            if return_code:
                print_yellow(
                    "WARNING: Running the post-deploy hook failed with return code %s"
                    % return_code,
                    newline_before=True,
                    newline_after=True)
                print_yellow(
                    "The tag %s has been rolled out.  However, you should run the post-deploy hook '%s' manually, or re-run this deployment."
                    % (tag, config.post_deploy),
                    newline_after=True)
                post_deploy_success = False

        if self.namespace is None:
            deployment_tag = "%s--%s" % (
                self.context,
                datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
            print_green("Updating ECR deployment tags in %s..." %
                        self.ecr.project_repo,
                        newline_after=True)
            try:
                self.ecr.retag(tag, self.context)
                print_green("Updated ECR tag %s -> %s" % (tag, self.context))

                self.ecr.retag(tag, deployment_tag)
                print_green("Updated ECR tag %s -> %s" % (tag, deployment_tag),
                            newline_after=True)
            except (ValueError, ClientError) as e:
                print_yellow(
                    "WARNING: Updating ECR deployment tags failed due to the error: '%s'"
                    % str(e),
                    newline_before=True,
                    newline_after=True)
                print_yellow(
                    "The tag %s has been rolled out.  However, you should create the ECR tags '%s' and '%s' manually, or re-run this deployment."
                    % (tag, deployment_tag, self.context),
                    newline_after=True)
                post_deploy_success = False

            remote = git_remote or config.git_remote
            if remote is not None:
                print_green("Pushing Git deployment tags to %s..." % remote,
                            newline_after=True)
                try:
                    shout("git fetch %s" % remote)
                    shout("git tag -f %s %s" % (self.context, tag),
                          print_output=True)
                    shout("git tag -f %s %s" % (deployment_tag, tag),
                          print_output=True)
                    shout("git push -f --no-verify %s refs/tags/%s" %
                          (remote, self.context),
                          print_output=True)
                    print_green("Updated Git tag %s -> %s" %
                                (tag, self.context))
                    shout("git push -f --no-verify %s refs/tags/%s" %
                          (remote, deployment_tag),
                          print_output=True)
                    print_green("Updated Git tag %s -> %s" %
                                (tag, deployment_tag),
                                newline_after=True)
                except CalledProcessError as e:
                    print_yellow(
                        "WARNING: Creating Git deployment tags failed due to the error: '%s'"
                        % str(e),
                        newline_before=True,
                        newline_after=True)
                    print_yellow(
                        "The tag %s has been rolled out.  However, you should create the Git tags '%s' and '%s' manually, or re-run this deployment."
                        % (tag, deployment_tag, self.context),
                        newline_after=True)
                    post_deploy_success = False

        if post_deploy_success:
            print_green("Deployment succeeded!")
        else:
            raise HokusaiError("One or more post-deploy steps failed!")
Beispiel #7
0
    def update(self,
               tag,
               constraint,
               git_remote,
               timeout,
               update_config=False,
               filename=None):
        if not self.ecr.project_repo_exists():
            raise HokusaiError("Project repo does not exist.  Aborting.")

        digest = self.ecr.image_digest_for_tag(tag)
        if digest is None:
            raise HokusaiError(
                "Could not find an image digest for tag %s.  Aborting." % tag)

        if self.namespace is None:
            print_green("Deploying %s to %s..." % (digest, self.context),
                        newline_after=True)
        else:
            print_green("Deploying %s to %s/%s..." %
                        (digest, self.context, self.namespace),
                        newline_after=True)
        """
    This logic should be refactored, but essentially if namespace and filename are provided, the caller is
    a review app, while if namespace is None it is either staging or production.  If filename is unset for staging
    or production it is targeting the 'canonical' app, i.e. staging.yml or production.yml while if it is set it is
    trageting a 'canary' app.

    For the canonical app, run deploy hooks and post-depoy steps creating deployment tags
    For a canary app, skip deploy hooks and post-deploy steps
    For review apps, run deploy hooks but skip post-deploy steps

    For all deployment rollouts, if update_config or filename targets a yml file, bust the
    deployment cache using k8s field selectors and get deployments to watch the rollout from
    the yml file spec
    """

        # Run the pre-deploy hook for the canonical app or a review app
        if config.pre_deploy and (filename is None or
                                  (filename and self.namespace)):
            print_green("Running pre-deploy hook '%s'..." % config.pre_deploy,
                        newline_after=True)
            return_code = CommandRunner(self.context,
                                        namespace=self.namespace).run(
                                            tag,
                                            config.pre_deploy,
                                            constraint=constraint,
                                            tty=False)
            if return_code:
                raise HokusaiError(
                    "Pre-deploy hook failed with return code %s" % return_code,
                    return_code=return_code)

        # Patch the deployments
        deployment_timestamp = datetime.datetime.utcnow().strftime("%s%f")

        if filename is None:
            kubernetes_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR,
                                          "%s.yml" % self.context)
        else:
            kubernetes_yml = filename

        # If a review app, a canary app or the canonical app while updating config,
        # bust the deployment cache and populate deployments from the yaml file
        if filename or update_config:
            self.cache = []
            for item in yaml.safe_load_all(open(kubernetes_yml, 'r')):
                if item['kind'] == 'Deployment':
                    self.cache.append(item)

        # If updating config, path the spec and apply
        if update_config:
            print_green(
                "Patching Deployments in spec %s with image digest %s" %
                (kubernetes_yml, digest),
                newline_after=True)
            payload = []
            for item in yaml.safe_load_all(open(kubernetes_yml, 'r')):
                if item['kind'] == 'Deployment':
                    item['spec']['template']['metadata']['labels'][
                        'deploymentTimestamp'] = deployment_timestamp
                    item['spec']['progressDeadlineSeconds'] = timeout
                    for container in item['spec']['template']['spec'][
                            'containers']:
                        if self.ecr.project_repo in container['image']:
                            container['image'] = "%s@%s" % (
                                self.ecr.project_repo, digest)
                payload.append(item)

            f = NamedTemporaryFile(delete=False)
            f.write(YAML_HEADER)
            f.write(yaml.safe_dump_all(payload, default_flow_style=False))
            f.close()

            print_green("Applying patched spec %s..." % f.name,
                        newline_after=True)
            try:
                shout(self.kctl.command("apply -f %s" % f.name),
                      print_output=True)
            finally:
                os.unlink(f.name)

        # If not updating config, patch the deployments in the cache and call kubectl patch to update
        else:
            for deployment in self.cache:
                containers = [(container['name'], container['image'])
                              for container in deployment['spec']['template']
                              ['spec']['containers']]
                deployment_targets = [{
                    "name":
                    name,
                    "image":
                    "%s@%s" % (self.ecr.project_repo, digest)
                } for name, image in containers
                                      if self.ecr.project_repo in image]
                patch = {
                    "spec": {
                        "template": {
                            "metadata": {
                                "labels": {
                                    "deploymentTimestamp": deployment_timestamp
                                }
                            },
                            "spec": {
                                "containers": deployment_targets
                            }
                        },
                        "progressDeadlineSeconds": timeout
                    }
                }

                print_green("Patching deployment %s..." %
                            deployment['metadata']['name'],
                            newline_after=True)
                shout(
                    self.kctl.command(
                        "patch deployment %s -p '%s'" %
                        (deployment['metadata']['name'], json.dumps(patch))))

        # Watch the rollouts in the cache and if any fail, roll back
        print_green("Waiting for deployment rollouts to complete...")
        rollout_commands = [
            self.kctl.command("rollout status deployment/%s" %
                              deployment['metadata']['name'])
            for deployment in self.cache
        ]
        return_codes = shout_concurrent(rollout_commands, print_output=True)
        if any(return_codes):
            print_red(
                "One or more deployment rollouts failed!  Rolling back...",
                newline_before=True,
                newline_after=True)
            rollback_commands = [
                self.kctl.command("rollout undo deployment/%s" %
                                  deployment['metadata']['name'])
                for deployment in self.cache
            ]
            shout_concurrent(rollback_commands, print_output=True)
            raise HokusaiError("Deployment failed!")

        post_deploy_success = True

        # Run the post-deploy hook for the canonical app or a review app
        if config.post_deploy and (filename is None or
                                   (filename and self.namespace)):
            print_green("Running post-deploy hook '%s'..." %
                        config.post_deploy,
                        newline_after=True)
            return_code = CommandRunner(self.context,
                                        namespace=self.namespace).run(
                                            tag,
                                            config.post_deploy,
                                            constraint=constraint,
                                            tty=False)
            if return_code:
                print_yellow(
                    "WARNING: Running the post-deploy hook failed with return code %s"
                    % return_code,
                    newline_before=True,
                    newline_after=True)
                print_yellow(
                    "The image digest %s has been rolled out.  However, you should run the post-deploy hook '%s' manually, or re-run this deployment."
                    % (digest, config.post_deploy),
                    newline_after=True)
                post_deploy_success = False

        # For the canonical app, create tags
        if filename is None:
            deployment_tag = "%s--%s" % (
                self.context,
                datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
            print_green("Updating ECR deployment tags in %s..." %
                        self.ecr.project_repo,
                        newline_after=True)
            try:
                self.ecr.retag(tag, self.context)
                print_green("Updated ECR tag %s -> %s" % (tag, self.context))

                self.ecr.retag(tag, deployment_tag)
                print_green("Updated ECR tag %s -> %s" % (tag, deployment_tag),
                            newline_after=True)
            except (ValueError, ClientError) as e:
                print_yellow(
                    "WARNING: Updating ECR deployment tags failed due to the error: '%s'"
                    % str(e),
                    newline_before=True,
                    newline_after=True)
                print_yellow(
                    "The tag %s has been rolled out.  However, you should create the ECR tags '%s' and '%s' manually, or re-run this deployment."
                    % (tag, deployment_tag, self.context),
                    newline_after=True)
                post_deploy_success = False

            remote = git_remote or config.git_remote
            if remote:
                print_green("Pushing Git deployment tags to %s..." % remote,
                            newline_after=True)
                try:
                    shout("git fetch %s" % remote)
                    shout("git tag -f %s %s" % (self.context, tag),
                          print_output=True)
                    shout("git tag -f %s %s" % (deployment_tag, tag),
                          print_output=True)
                    shout("git push -f --no-verify %s refs/tags/%s" %
                          (remote, self.context),
                          print_output=True)
                    print_green("Updated Git tag %s -> %s" %
                                (tag, self.context))
                    shout("git push -f --no-verify %s refs/tags/%s" %
                          (remote, deployment_tag),
                          print_output=True)
                    print_green("Updated Git tag %s -> %s" %
                                (tag, deployment_tag),
                                newline_after=True)
                except CalledProcessError as e:
                    print_yellow(
                        "WARNING: Creating Git deployment tags failed due to the error: '%s'"
                        % str(e),
                        newline_before=True,
                        newline_after=True)
                    print_yellow(
                        "The tag %s has been rolled out.  However, you should create the Git tags '%s' and '%s' manually, or re-run this deployment."
                        % (tag, deployment_tag, self.context),
                        newline_after=True)
                    post_deploy_success = False

        if post_deploy_success:
            print_green("Deployment succeeded!")
        else:
            raise HokusaiError("One or more post-deploy steps failed!")
Beispiel #8
0
    def update(self, tag, constraint, git_remote, resolve_tag_sha1=True):
        if not self.ecr.project_repo_exists():
            raise HokusaiError("Project repo does not exist.  Aborting.")

        if resolve_tag_sha1:
            tag = self.ecr.find_git_sha1_image_tag(tag)
            if tag is None:
                raise HokusaiError(
                    "Could not find a git SHA1 for tag %s.  Aborting." % tag)

        if self.namespace is None:
            print_green("Deploying %s to %s..." % (tag, self.context))
        else:
            print_green("Deploying %s to %s/%s..." %
                        (tag, self.context, self.namespace))

        if self.namespace is None:
            self.ecr.retag(tag, self.context)
            print_green("Updated tag %s -> %s" % (tag, self.context))

            deployment_tag = "%s--%s" % (
                self.context,
                datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
            self.ecr.retag(tag, deployment_tag)
            print_green("Updated tag %s -> %s" % (tag, deployment_tag))

            if git_remote is not None:
                print_green("Pushing deployment tags to %s..." % git_remote)
                shout("git tag -f %s" % self.context, print_output=True)
                shout("git tag %s" % deployment_tag, print_output=True)
                shout("git push --force %s --tags" % git_remote,
                      print_output=True)

        if config.pre_deploy is not None:
            print_green("Running pre-deploy hook '%s'..." % config.pre_deploy)
            return_code = CommandRunner(self.context,
                                        namespace=self.namespace).run(
                                            tag,
                                            config.pre_deploy,
                                            constraint=constraint)
            if return_code:
                raise HokusaiError(
                    "Pre-deploy hook failed with return code %s" % return_code,
                    return_code=return_code)

        deployment_timestamp = datetime.datetime.utcnow().strftime("%s%f")
        for deployment in self.cache:
            containers = deployment['spec']['template']['spec']['containers']
            container_names = [container['name'] for container in containers]
            deployment_targets = [{
                "name":
                name,
                "image":
                "%s:%s" % (self.ecr.project_repo, tag)
            } for name in container_names]
            patch = {
                "spec": {
                    "template": {
                        "metadata": {
                            "labels": {
                                "deploymentTimestamp": deployment_timestamp
                            }
                        },
                        "spec": {
                            "containers": deployment_targets
                        }
                    }
                }
            }
            print_green("Patching deployment %s..." %
                        deployment['metadata']['name'])
            shout(
                self.kctl.command(
                    "patch deployment %s -p '%s'" %
                    (deployment['metadata']['name'], json.dumps(patch))))

        print_green("Waiting for rollout to complete...")

        rollout_commands = [
            self.kctl.command("rollout status deployment/%s" %
                              deployment['metadata']['name'])
            for deployment in self.cache
        ]
        return_code = shout_concurrent(rollout_commands)
        if return_code:
            raise HokusaiError("Deployment failed!", return_code=return_code)

        if config.post_deploy is not None:
            print_green("Running post-deploy hook '%s'..." %
                        config.post_deploy)
            return_code = CommandRunner(self.context,
                                        namespace=self.namespace).run(
                                            tag,
                                            config.post_deploy,
                                            constraint=constraint)
            if return_code:
                raise HokusaiError(
                    "Post-deploy hook failed with return code %s" %
                    return_code,
                    return_code=return_code)
Beispiel #9
0
    def update(self,
               tag,
               constraint,
               git_remote,
               timeout,
               resolve_tag_sha1=True,
               update_config=False,
               filename=None):
        if not self.ecr.project_repo_exists():
            raise HokusaiError("Project repo does not exist.  Aborting.")

        if resolve_tag_sha1:
            tag = self.ecr.find_git_sha1_image_tag(tag)
            if tag is None:
                raise HokusaiError(
                    "Could not find a git SHA1 for tag %s.  Aborting." % tag)

        if self.namespace is None:
            print_green("Deploying %s to %s..." % (tag, self.context),
                        newline_after=True)
        else:
            print_green("Deploying %s to %s/%s..." %
                        (tag, self.context, self.namespace),
                        newline_after=True)

        if config.pre_deploy is not None:
            print_green("Running pre-deploy hook '%s'..." % config.pre_deploy,
                        newline_after=True)
            return_code = CommandRunner(self.context,
                                        namespace=self.namespace).run(
                                            tag,
                                            config.pre_deploy,
                                            constraint=constraint,
                                            tty=False)
            if return_code:
                raise HokusaiError(
                    "Pre-deploy hook failed with return code %s" % return_code,
                    return_code=return_code)

        deployment_timestamp = datetime.datetime.utcnow().strftime("%s%f")

        if update_config:
            if filename is None:
                kubernetes_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR,
                                              "%s.yml" % self.context)
            else:
                kubernetes_yml = filename

            print_green("Patching Deployments in spec %s with tag %s" %
                        (kubernetes_yml, tag),
                        newline_after=True)
            payload = []
            for item in yaml.safe_load_all(open(kubernetes_yml, 'r')):
                if item['kind'] == 'Deployment':
                    item['spec']['template']['metadata']['labels'][
                        'deploymentTimestamp'] = deployment_timestamp
                    item['spec']['progressDeadlineSeconds'] = timeout
                    for container in item['spec']['template']['spec'][
                            'containers']:
                        if self.ecr.project_repo in container['image']:
                            container['image'] = "%s:%s" % (
                                self.ecr.project_repo, tag)
                payload.append(item)

            f = NamedTemporaryFile(delete=False)
            f.write(YAML_HEADER)
            f.write(yaml.safe_dump_all(payload, default_flow_style=False))
            f.close()

            print_green("Applying patched spec %s..." % f.name,
                        newline_after=True)
            try:
                shout(self.kctl.command("apply -f %s" % f.name),
                      print_output=True)
            finally:
                os.unlink(f.name)

        else:
            for deployment in self.cache:
                containers = [(container['name'], container['image'])
                              for container in deployment['spec']['template']
                              ['spec']['containers']]
                deployment_targets = [{
                    "name":
                    name,
                    "image":
                    "%s:%s" % (self.ecr.project_repo, tag)
                } for name, image in containers
                                      if self.ecr.project_repo in image]
                patch = {
                    "spec": {
                        "template": {
                            "metadata": {
                                "labels": {
                                    "deploymentTimestamp": deployment_timestamp
                                }
                            },
                            "spec": {
                                "containers": deployment_targets
                            }
                        },
                        "progressDeadlineSeconds": timeout
                    }
                }

                print_green("Patching deployment %s..." %
                            deployment['metadata']['name'],
                            newline_after=True)
                shout(
                    self.kctl.command(
                        "patch deployment %s -p '%s'" %
                        (deployment['metadata']['name'], json.dumps(patch))))

        print_green("Waiting for deployment rollouts to complete...")

        rollout_commands = [
            self.kctl.command("rollout status deployment/%s" %
                              deployment['metadata']['name'])
            for deployment in self.cache
        ]
        return_codes = shout_concurrent(rollout_commands, print_output=True)
        if any(return_codes):
            print_red(
                "One or more deployment rollouts timed out!  Rolling back...",
                newline_before=True,
                newline_after=True)
            rollback_commands = [
                self.kctl.command("rollout undo deployment/%s" %
                                  deployment['metadata']['name'])
                for deployment in self.cache
            ]
            shout_concurrent(rollback_commands, print_output=True)
            raise HokusaiError("Deployment failed!")

        post_deploy_success = True

        if config.post_deploy is not None:
            print_green("Running post-deploy hook '%s'..." %
                        config.post_deploy,
                        newline_after=True)
            return_code = CommandRunner(self.context,
                                        namespace=self.namespace).run(
                                            tag,
                                            config.post_deploy,
                                            constraint=constraint,
                                            tty=False)
            if return_code:
                print_yellow(
                    "WARNING: Running the post-deploy hook failed with return code %s"
                    % return_code,
                    newline_before=True,
                    newline_after=True)
                print_yellow(
                    "The tag %s has been rolled out.  However, you should run the post-deploy hook '%s' manually, or re-run this deployment."
                    % (tag, config.post_deploy),
                    newline_after=True)
                post_deploy_success = False

        if self.namespace is None:
            deployment_tag = "%s--%s" % (
                self.context,
                datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
            print_green("Updating ECR deployment tags in %s..." %
                        self.ecr.project_repo,
                        newline_after=True)
            try:
                self.ecr.retag(tag, self.context)
                print_green("Updated ECR tag %s -> %s" % (tag, self.context))

                self.ecr.retag(tag, deployment_tag)
                print_green("Updated ECR tag %s -> %s" % (tag, deployment_tag),
                            newline_after=True)
            except (ValueError, ClientError) as e:
                print_yellow(
                    "WARNING: Updating ECR deployment tags failed due to the error: '%s'"
                    % str(e),
                    newline_before=True,
                    newline_after=True)
                print_yellow(
                    "The tag %s has been rolled out.  However, you should create the ECR tags '%s' and '%s' manually, or re-run this deployment."
                    % (tag, deployment_tag, self.context),
                    newline_after=True)
                post_deploy_success = False

            remote = git_remote or config.git_remote
            if remote is not None:
                print_green("Pushing Git deployment tags to %s..." % remote,
                            newline_after=True)
                try:
                    shout("git fetch %s" % remote)
                    shout("git tag -f %s %s" % (self.context, tag),
                          print_output=True)
                    shout("git tag -f %s %s" % (deployment_tag, tag),
                          print_output=True)
                    shout("git push -f --no-verify %s refs/tags/%s" %
                          (remote, self.context),
                          print_output=True)
                    print_green("Updated Git tag %s -> %s" %
                                (tag, self.context))
                    shout("git push -f --no-verify %s refs/tags/%s" %
                          (remote, deployment_tag),
                          print_output=True)
                    print_green("Updated Git tag %s -> %s" %
                                (tag, deployment_tag),
                                newline_after=True)
                except CalledProcessError as e:
                    print_yellow(
                        "WARNING: Creating Git deployment tags failed due to the error: '%s'"
                        % str(e),
                        newline_before=True,
                        newline_after=True)
                    print_yellow(
                        "The tag %s has been rolled out.  However, you should create the Git tags '%s' and '%s' manually, or re-run this deployment."
                        % (tag, deployment_tag, self.context),
                        newline_after=True)
                    post_deploy_success = False

        if post_deploy_success:
            print_green("Deployment succeeded!")
        else:
            raise HokusaiError("One or more post-deploy steps failed!")