Esempio n. 1
0
class Application(object):
    """
    Create an Application which maps to a kubernetes namespace
    """
    def __init__(self, name, client=None):
        self.name = name
        if client is None:
            self._client = KubernetesApiClient(use_proxy=True)
        else:
            self._client = client

        self._registry_spec = None
        self._software_info = SoftwareInfo()
        if self._software_info.registry_is_private():
            secret = KubeObjectConfigFile(
                DEFAULT_SECRET_YAML_PATH,
                {"REGISTRY_SECRETS": self._software_info.registry_secrets})
            for obj in secret.get_swagger_objects():
                if isinstance(obj, swagger_client.V1Secret):
                    self._registry_spec = obj
            assert self._registry_spec, "Argo registry specification is missing"

        self._am_service_spec = None
        self._am_deployment_spec = None

        # AA-2471: Hack to add AXOPS_EXT_DNS to Application Manager
        elb = InternalRoute("axops", "axsys", client=self._client)
        elb_status = elb.status(with_loadbalancer_info=True)["loadbalancer"][0]
        if not elb_status:
            raise AXPlatformException(
                "Could not get axops elb address {}".format(elb_status))

        replacements = {
            "NAMESPACE": self._software_info.image_namespace,
            "VERSION": self._software_info.image_version,
            "REGISTRY": self._software_info.registry,
            "APPLICATION_NAME": self.name,
            "AXOPS_EXT_DNS": elb_status
        }
        cluster_name_id = os.getenv("AX_CLUSTER_NAME_ID", None)
        assert cluster_name_id, "Cluster name id is None!"
        cluster_config = AXClusterConfig(cluster_name_id=cluster_name_id)
        if cluster_config.get_cluster_provider() != ClusterProvider.USER:
            axam_path = DEFAULT_AM_YAML_PATH
        else:
            axam_path = "/ax/config/service/argo-all/axam-svc.yml.in"
            replacements["ARGO_DATA_BUCKET_NAME"] = os.getenv(
                "ARGO_DATA_BUCKET_NAME")

        logger.info("Using replacements: %s", replacements)

        k = KubeObjectConfigFile(axam_path, replacements)
        for obj in k.get_swagger_objects():
            if isinstance(obj, swagger_client.V1Service):
                self._am_service_spec = obj
            elif isinstance(obj, swagger_client.V1beta1Deployment):
                self._am_deployment_spec = obj
                self._add_pod_metadata("deployment",
                                       self._am_deployment_spec.metadata.name,
                                       is_label=True)
                self._add_pod_metadata(
                    "ax_costid",
                    json.dumps({
                        "app": self.name,
                        "service": "axam-deployment",
                        "user": "******"
                    }))
            else:
                logger.debug("Ignoring specification of type {}".format(
                    type(obj)))
        assert self._am_service_spec and self._am_deployment_spec, "Application monitor specification is missing"

    def _add_pod_metadata(self, key, value, is_label=False):
        """
        Helper function to add metadata to deployment pod spec for AXAM
        """
        pod_meta = self._am_deployment_spec.spec.template.metadata
        if is_label:
            if pod_meta.labels is None:
                pod_meta.labels = {}
            pod_meta.labels[key] = value
        else:
            if pod_meta.annotations is None:
                pod_meta.annotations = {}
            pod_meta.annotations[key] = value

    def create(self, force_recreate=False):
        """
        Create a kubernetes namespace and populate it with argo registry

        Idempotency: This function will be idempotent as long as the content
        of the secret is not changed. If create is called with a registry secret
        that has been updated and the namespace with the secret already exists
        then it will not update the secret for now.
        """
        @retry_not_exists
        def create_ns_in_provider():
            namespace = swagger_client.V1Namespace()
            namespace.metadata = swagger_client.V1ObjectMeta()
            namespace.metadata.name = self.name
            self._client.api.create_namespace(namespace)

        # NOTE: 403 is not retried as application is getting deleted in parallel
        # 422 is unprocessable object (aka error in spec)
        @retry_unless(status_code=[403, 422])
        def create_reg_in_provider():
            if self._registry_spec is None:
                return
            try:
                self._client.api.create_namespaced_secret(
                    self._registry_spec, self.name)
            except swagger_client.rest.ApiException as e:
                if e.status == 409:
                    self._client.api.patch_namespaced_secret(
                        self._registry_spec.to_dict(), self.name,
                        self._registry_spec.metadata.name)
                else:
                    raise e

        @retry_unless(status_code=[403, 422])
        def create_app_monitor_service_in_provider():
            try:
                self._client.api.create_namespaced_service(
                    self._am_service_spec, self.name)
            except swagger_client.rest.ApiException as e:
                if e.status == 409:
                    self._client.api.patch_namespaced_service(
                        self._am_service_spec.to_dict(), self.name,
                        self._am_service_spec.metadata.name)
                else:
                    raise e

        @retry_unless(status_code=[403, 422])
        def create_app_monitor_deployment_in_provider():
            try:
                self._client.apisappsv1beta1_api.create_namespaced_deployment(
                    self._am_deployment_spec, self.name)
            except swagger_client.rest.ApiException as e:
                if e.status == 409:
                    if force_recreate:
                        # add a new metadata in pod spec to force the recreation of pods
                        self._add_pod_metadata(
                            "applatix.io/force-recreate-salt",
                            str(uuid.uuid4()))

                    self._client.apisappsv1beta1_api.replace_namespaced_deployment(
                        self._am_deployment_spec, self.name,
                        self._am_deployment_spec.metadata.name)
                else:
                    raise e

        try:
            logger.debug("Creating application {}".format(self.name))
            create_ns_in_provider()
            logger.debug("Created namespace {}".format(self.name))
            create_reg_in_provider()
            create_app_monitor_service_in_provider()
            logger.debug("Created application monitor service {}".format(
                self._am_service_spec.metadata.name))
            create_app_monitor_deployment_in_provider()
            logger.debug("Created application monitor deployment {}".format(
                self._am_deployment_spec.metadata.name))
        except Exception as e:
            logger.exception(e)

    def delete(self, timeout=None):
        """
        Delete a kubernetes namespace and image secret for Argo

        Idempotency: Can be repeatedly called
        """
        delete_grace_period = 1
        options = swagger_client.V1DeleteOptions()
        options.grace_period_seconds = delete_grace_period
        options.orphan_dependents = False

        @retry_unless(swallow_code=[404, 409])
        def delete_ns_in_provider():
            """
            The retry is not done for 404 (not found) and also for 409 (conflict)
            The 404 case is for simple retry. 409 happens when application delete was
            requested but not complete and another request came in.
            """
            logger.debug("Deleting application {}".format(self.name))
            self._client.api.delete_namespace(options, self.name)

        delete_ns_in_provider()

        start_time = time.time()
        while self.exists():
            logger.debug("Application {} still exists".format(self.name))
            time.sleep(delete_grace_period + 1)
            wait_time = int(time.time() - start_time)
            if timeout is not None and wait_time > timeout:
                raise AXTimeoutException(
                    "Could not delete namespace {} in {} seconds".format(
                        self.name, timeout))

    def exists(self):
        @retry_unless_not_found
        def get_ns_in_provider():
            try:
                stat = self._client.api.read_namespace(self.name)
                return True
            except swagger_client.rest.ApiException as e:
                if e.status == 404:
                    return False
                else:
                    raise e

        return get_ns_in_provider()

    def status(self):
        """
        This function checks the following:
        1. Namespace exists?
        2. Argo Registry exists?
        3. TODO: Application Monitor exists
        Returns:
            A json dict with the status of each
            {
                'namespace': True/False,
                'registry': True/False,
                'monitor': True/False
            }
        """
        ret = {'namespace': False, 'registry': False, 'monitor': False}

        if not self.exists():
            return ret

        ret['namespace'] = True
        ns = self._get_registry_from_provider()
        if ns is None:
            return ret

        ret['registry'] = True
        srv = self._get_am_service_from_provider()
        if srv is None:
            return ret

        am_dep = self._get_am_deployment_from_provider()
        if am_dep is not None and am_dep.status.available_replicas == am_dep.status.replicas:
            ret["monitor"] = True

        return ret

    def healthy(self):
        """
        If all components are present/healthy then return True
        else return False
        """
        d = self.status()
        for component in d:
            if not d[component]:
                return False
        return True

    def events(self, name=None):
        return self._get_events_from_provider(name).items

    @retry_unless(swallow_code=[404])
    def _get_registry_from_provider(self):
        if self._registry_spec is not None:
            return self._client.api.read_namespaced_secret(
                self.name, self._registry_spec.metadata.name)
        else:
            return "NotNeeded"

    @retry_unless(swallow_code=[404])
    def _get_am_service_from_provider(self):
        return self._client.api.read_namespaced_service(
            self.name, self._am_service_spec.metadata.name)

    @retry_unless(swallow_code=[404])
    def _get_am_deployment_from_provider(self):
        return self._client.apisappsv1beta1_api.read_namespaced_deployment(
            self.name, self._am_deployment_spec.metadata.name)

    @retry_unless(swallow_code=[404])
    def _get_events_from_provider(self, name):
        # XXX: For some reason list_namespaced_event does not take a namespace but the _21 version
        #      of the function does. Hopefully this gets fixed in swagger soon
        field_selector = None
        if name is not None:
            field_selector = "involvedObject.name={}".format(name)
        return self._client.api.list_namespaced_event(
            self.name, field_selector=field_selector)
Esempio n. 2
0
class Container(KubeObject):
    """
    Class for creating container specifications
    """

    LIVENESS_PROBE = 1
    READINESS_PROBE = 2

    def __init__(self, name, image, pull_policy=None):
        """
        Construct a container that will provide the spec for a kubernetes container
        http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_container
        Args:
            name: name of a container. must be conformant to kubernetes container name
            image: image for container
            pull_policy: pull policy based on kubernetes. If None then kubernetes default is used
        """
        self.name = name
        self.image = image
        self.image_pull_policy = pull_policy

        self.command = None
        self.args = None

        self.vmap = {}
        self.env_map = {}
        self.ports = []

        self.resources = None
        self.privileged = None

        self.software_info = SoftwareInfo()
        self.probes = {}

    def generate_spec(self):
        c = swagger_client.V1Container()
        c.name = self.name
        c.image = self.image

        if self.resources is not None:
            c.resources = swagger_client.V1ResourceRequirements()
            c.resources.requests = {}
            c.resources.limits = {}
            if "cpu_cores" in self.resources:
                c.resources.requests["cpu"] = str(
                    self.resources["cpu_cores"][0])
                if self.resources["cpu_cores"][1] is not None:
                    c.resources.limits["cpu"] = str(
                        self.resources["cpu_cores"][1])

            if "mem_mib" in self.resources:
                c.resources.requests["memory"] = "{}Mi".format(
                    self.resources["mem_mib"][0])
                if self.resources["mem_mib"][1] is not None:
                    c.resources.limits["memory"] = "{}Mi".format(
                        self.resources["mem_mib"][1])

        # Kubernetes 1.5 requires init container must specify image pull policy. Since we are setting
        # a pull policy for all containers, we want to replicate the kubernetes default behavior of pulling
        # the image if tag is "latest"
        if self.image.endswith(':latest'):
            c.image_pull_policy = ContainerImagePullPolicy.PullAlways
        else:
            c.image_pull_policy = self.image_pull_policy or ContainerImagePullPolicy.PullIfNotPresent

        if self.command:
            c.command = self.command
        if self.args:
            c.args = self.args

        c.volume_mounts = []
        for _, vol in self.vmap.iteritems():
            c.volume_mounts.append(vol.get_container_spec())

        c.env = []
        for _, env in self.env_map.iteritems():
            c.env.append(env)

        if self.privileged is not None:
            c.security_context = swagger_client.V1SecurityContext()
            c.security_context.privileged = self.privileged

        for probe in self.probes:
            probe_spec = self.probes[probe]
            probe_k8s_spec = Container._generate_probe_spec(probe_spec)
            if probe == Container.LIVENESS_PROBE:
                c.liveness_probe = probe_k8s_spec
            elif probe == Container.READINESS_PROBE:
                c.readiness_probe = probe_k8s_spec
            else:
                raise AXIllegalArgumentException(
                    "Unexpected probe type {} found with spec {}".format(
                        probe, probe_spec))

        return c

    def add_resource_constraints(self, resource, request, limit=None):
        if self.resources is None:
            self.resources = {}
        self.resources[resource] = (request, limit)

    def add_volume(self, volume):
        self.vmap[volume.name] = volume

    def add_volumes(self, volumes):
        for vol in volumes or []:
            self.add_volume(vol)

    def get_volume(self, name):
        return self.vmap.get(name, None)

    def add_env(self, name, value=None, value_from=None):
        env = swagger_client.V1EnvVar()
        env.name = name
        if value is not None:
            env.value = value
        else:
            assert value_from is not None, "value and value_from both cannot be None for env {}".format(
                name)
            env.value_from = swagger_client.V1EnvVarSource()
            env.value_from.field_ref = swagger_client.V1ObjectFieldSelector()
            env.value_from.field_ref.field_path = value_from
            # Some 1.5 requires this. https://github.com/kubernetes/kubernetes/issues/39189
            env.value_from.field_ref.api_version = "v1"

        self.env_map[name] = env

    def add_probe(self, probe_type, probe_spec):
        self.probes[probe_type] = probe_spec

    def parse_probe_spec(self, container_template):
        """
        @type container_template: argo.template.v1.container.ContainerTemplate
        """
        if container_template.liveness_probe:
            probe_type = Container.LIVENESS_PROBE
            self.add_probe(probe_type, container_template.liveness_probe)
        if container_template.readiness_probe:
            probe_type = Container.READINESS_PROBE
            self.add_probe(probe_type, container_template.readiness_probe)

    def get_registry(self, namespace="axuser"):
        """
        This function returns the name of the secrets file that needs to be
        used in the pod specification image_pull_secrets array
        """
        (reg, _, _) = DockerImage(fullname=self.image).docker_names()
        if reg == self.software_info.registry:
            if self.software_info.registry_is_private():
                return "applatix-registry"
            else:
                return None
        else:
            try:
                smanager = SecretsManager()
                secret = smanager.get_imgpull(reg, namespace)
                if secret:
                    return secret.metadata.name

                # Code for copying the registry to the app namespace if
                # it does not exist. We do not copy to axuser as secrets
                # are always created there.
                secret_axuser = smanager.get_imgpull(reg, "axuser")
                if secret_axuser and namespace != "axuser":
                    smanager.copy_imgpull(secret_axuser, namespace)
                    return secret_axuser.metadata.name
            except Exception as e:
                logger.debug(
                    "Did not find a secret for registry {} due to exception {}"
                    .format(reg, e))
            return None

    def volume_iterator(self):
        for _, vol in self.vmap.iteritems():
            yield vol

    @staticmethod
    def _generate_probe_spec(spec):
        """
        @type spec argo.template.v1.container.ContainerProbe
        """
        try:
            probe = swagger_client.V1Probe()
            probe.initial_delay_seconds = spec.initial_delay_seconds
            probe.timeout_seconds = spec.timeout_seconds
            probe.period_seconds = spec.period_seconds
            probe.failure_threshold = spec.failure_threshold
            probe.success_threshold = spec.success_threshold

            if spec.exec_probe:
                action = swagger_client.V1ExecAction()
                action.command = shlex.split(spec.exec_probe.command)
                probe._exec = action
                return probe
            elif spec.http_get:
                action = swagger_client.V1HTTPGetAction()
                action.path = spec.http_get.path
                action.port = spec.http_get.port
                headers = spec.http_get.http_headers
                action.http_headers = []
                for header in headers or []:
                    h = swagger_client.V1HTTPHeader()
                    h.name = header["name"]
                    h.value = header["value"]
                    action.http_headers.append(h)
                probe.http_get = action
                return probe
            else:
                logger.debug("Cannot handle probe {}".format(spec))
        except Exception as e:
            raise AXIllegalArgumentException(
                "Probe {} cannot be processed due to error {}".format(spec, e))

        return None
Esempio n. 3
0
class AXSYSKubeYamlUpdater(object):
    """
    This class loads a kubernetes yaml file, updates resource,
    and generate objects that kube_object.py can consume
    """
    def __init__(self, config_file_path):
        assert os.path.isfile(
            config_file_path), "Config file {} is not a file".format(
                config_file_path)
        self._config_file = config_file_path
        self._cluster_name_id = AXClusterId().get_cluster_name_id()
        self._cluster_config = AXClusterConfig(
            cluster_name_id=self._cluster_name_id)
        self.cpu_mult, self.mem_mult, self.disk_mult, \
            self.daemon_cpu_mult, self.daemon_mem_mult = self._get_resource_multipliers()
        self._swagger_components = []
        self._yaml_components = []
        self._updated_raw = ""

        # TODO: when we support config software info using a config file, need to figure out how that
        # file gets passed through, since SoftwareInfo is not a singleton
        self._software_info = SoftwareInfo()

        self._load_objects()
        self._load_raw()

    @property
    def updated_raw(self):
        return self._updated_raw

    @property
    def components_in_dict(self):
        return self._yaml_components

    @property
    def components_in_swagger(self):
        return self._swagger_components

    def _load_objects(self):
        with open(self._config_file, "r") as f:
            data = f.read()
        for c in yaml.load_all(data):
            swagger_obj = self._config_yaml(c)
            yaml_obj = ApiClient().sanitize_for_serialization(swagger_obj)
            self._swagger_components.append(swagger_obj)
            self._yaml_components.append(yaml_obj)

    def _load_raw(self):
        self._updated_raw = yaml.dump_all(self._yaml_components)

    def _get_resource_multipliers(self):
        """
        Resources in yaml templates need to be multiplied with these numbers
        :return: cpu_multiplier, mem_multiplier, disk_multiplier
        """
        # Getting cluster size from cluster config, in order to configure resources
        # There are 3 situations we will be using AXClusterConfig
        #   - During install, since the class is a singleton, it has all the values we need
        #     no need to download from s3
        #   - During upgrade, since we are exporting AWS_DEFAULT_PROFILE, we can download
        #     cluster config files from s3 to get the values
        #   - During job creation: the node axmon runs has the proper roles to access s3

        try:
            ax_node_max = int(self._cluster_config.get_asxys_node_count())
            ax_node_type = self._cluster_config.get_axsys_node_type()
            usr_node_max = int(
                self._cluster_config.get_max_node_count()) - ax_node_max
            usr_node_type = self._cluster_config.get_axuser_node_type()
            assert all(
                [ax_node_max, ax_node_type, usr_node_max, usr_node_type])
        except Exception as e:
            logger.error(
                "Unable to read cluster config, skip resource config for %s. Error %s",
                self._config_file, e)
            return 1, 1, 1, 1, 1

        rc = AXSYSResourceConfig(
            ax_node_type=ax_node_type,
            ax_node_max=ax_node_max,
            usr_node_type=usr_node_type,
            usr_node_max=usr_node_max,
            cluster_type=self._cluster_config.get_ax_cluster_type())
        #logger.info("With %s %s axsys nodes, %s %s axuser nodes, component %s uses multipliers (%s, %s, %s, %s, %s)",
        #            ax_node_max, ax_node_type, usr_node_max, usr_node_type, self._config_file,
        #            rc.cpu_multiplier, rc.mem_multiplier, rc.disk_multiplier,
        #            rc.daemon_cpu_multiplier, rc.daemon_mem_multiplier)
        return rc.cpu_multiplier, rc.mem_multiplier, rc.disk_multiplier, rc.daemon_cpu_multiplier, rc.daemon_mem_multiplier

    def _config_yaml(self, kube_yaml_obj):
        """
        Load dict into swagger object, patch resource,
        sanitize, return a dict
        :param kube_yaml_obj:
        :return: swagger object with resource values finalized
        """
        kube_kind = kube_yaml_obj["kind"]
        (swagger_class_literal,
         swagger_instance) = KubeKindToV1KubeSwaggerObject[kube_kind]
        swagger_obj = ApiClient()._ApiClient__deserialize(
            kube_yaml_obj, swagger_class_literal)
        assert isinstance(swagger_obj, swagger_instance), \
            "{} has instance {}, expected {}".format(swagger_obj, type(swagger_obj), swagger_instance)

        if isinstance(swagger_obj, V1beta1Deployment):
            if not self._software_info.registry_is_private():
                swagger_obj.spec.template.spec.image_pull_secrets = None

            node_selector = swagger_obj.spec.template.spec.node_selector
            if node_selector.get('ax.tier', 'applatix') == 'master':
                # Skip updating containers on master.
                logger.info(
                    "Skip updating cpu, mem multipliers for pods on master: %s",
                    swagger_obj.metadata.name)
            else:
                for container in swagger_obj.spec.template.spec.containers:
                    self._update_container(container)
            return swagger_obj
        elif isinstance(swagger_obj, V1Pod):
            if not self._software_info.registry_is_private():
                swagger_obj.spec.image_pull_secrets = None
            return swagger_obj
        elif isinstance(swagger_obj, V1beta1DaemonSet):
            if not self._software_info.registry_is_private():
                swagger_obj.spec.template.spec.image_pull_secrets = None
            for container in swagger_obj.spec.template.spec.containers:
                # We are special-casing applet DaemonSet to compromise the fact that
                # we are using different node type for compute-intense nodes
                if swagger_obj.metadata.name == "applet":
                    self._update_container(container=container,
                                           is_daemon=True,
                                           update_resource=True)
                else:
                    self._update_container(container=container,
                                           is_daemon=True,
                                           update_resource=False)
            return swagger_obj
        elif isinstance(swagger_obj, V1beta1StatefulSet):
            if not self._software_info.registry_is_private():
                swagger_obj.spec.template.spec.image_pull_secrets = None
            return self._update_statefulset(swagger_obj)
        elif isinstance(swagger_obj, V1PersistentVolumeClaim):
            self._update_volume(swagger_obj)
            return swagger_obj
        else:
            # logger.info("Object %s does not need to configure resource", type(swagger_obj))
            # HACK, as the original hook will be messed up
            if isinstance(swagger_obj, V1Service):
                if swagger_obj.metadata.name == "axops":
                    swagger_obj.spec.load_balancer_source_ranges = []
                    for cidr in self._cluster_config.get_trusted_cidr():
                        # Seems swagger client does not support unicode ... SIGH
                        swagger_obj.spec.load_balancer_source_ranges.append(
                            str(cidr))

                # HACK #2: if we don't do this, kubectl will complain about something such as
                #
                # spec.ports[0].targetPort: Invalid value: "81": must contain at least one letter (a-z)
                #
                # p.target_port is defined as string though, but if its really a string, kubectl
                # is looking for a port name, rather than a number
                # SIGH ...
                for p in swagger_obj.spec.ports or []:
                    try:
                        p.target_port = int(p.target_port)
                    except (ValueError, TypeError):
                        pass
            return swagger_obj

    def _update_deployment_or_daemonset(self, kube_obj):
        assert isinstance(kube_obj, V1beta1Deployment) or isinstance(
            kube_obj, V1beta1DaemonSet)
        for container in kube_obj.spec.template.spec.containers:
            self._update_container(container)
        return kube_obj

    def _update_statefulset(self, kube_obj):
        assert isinstance(kube_obj, V1beta1StatefulSet)
        for container in kube_obj.spec.template.spec.containers:
            self._update_container(container)
        if isinstance(kube_obj.spec.volume_claim_templates, list):
            for vol in kube_obj.spec.volume_claim_templates:
                self._update_volume(vol)
        return kube_obj

    def _update_container(self,
                          container,
                          is_daemon=False,
                          update_resource=True):
        assert isinstance(container, V1Container)

        if update_resource:
            cpulim = container.resources.limits.get("cpu")
            memlim = container.resources.limits.get("memory")
            cpureq = container.resources.requests.get("cpu")
            memreq = container.resources.requests.get("memory")

            def _massage_cpu(orig):
                return orig * self.daemon_cpu_mult if is_daemon else orig * self.cpu_mult

            def _massage_mem(orig):
                return orig * self.daemon_mem_mult if is_daemon else orig * self.mem_mult

            if cpulim:
                rvc = ResourceValueConverter(value=cpulim, target="cpu")
                rvc.massage(_massage_cpu)
                container.resources.limits["cpu"] = "{}m".format(
                    rvc.convert("m"))
            if cpureq:
                rvc = ResourceValueConverter(value=cpureq, target="cpu")
                rvc.massage(_massage_cpu)
                container.resources.requests["cpu"] = "{}m".format(
                    rvc.convert("m"))
            if memlim:
                rvc = ResourceValueConverter(value=memlim, target="memory")
                rvc.massage(_massage_mem)
                container.resources.limits["memory"] = "{}Mi".format(
                    int(rvc.convert("Mi")))
            if memreq:
                rvc = ResourceValueConverter(value=memreq, target="memory")
                rvc.massage(_massage_mem)
                container.resources.requests["memory"] = "{}Mi".format(
                    int(rvc.convert("Mi")))

        if container.liveness_probe and container.liveness_probe.http_get:
            try:
                container.liveness_probe.http_get.port = int(
                    container.liveness_probe.http_get.port)
            except (ValueError, TypeError):
                pass
        if container.readiness_probe and container.readiness_probe.http_get:
            try:
                container.readiness_probe.http_get.port = int(
                    container.readiness_probe.http_get.port)
            except (ValueError, TypeError):
                pass

        # Add resource multiplier to containers in case we need them
        if not container.env:
            container.env = []
        container.env += self._generate_default_envs(is_daemon,
                                                     update_resource)

    def _update_volume(self, vol):
        assert isinstance(vol, V1PersistentVolumeClaim)
        vol_size = vol.spec.resources.requests["storage"]

        def _massage_disk(orig):
            return orig * self.disk_mult

        if vol_size:
            rvc = ResourceValueConverter(value=vol_size, target="storage")
            rvc.massage(_massage_disk)
            # Since AWS does not support value such as 1.5G, lets round up to its ceil
            vol.spec.resources.requests["storage"] = "{}Gi".format(
                int(ceil(rvc.convert("Gi"))))

        # Manually patch access mode as swagger client mistakenly interprets this as map
        vol.spec.access_modes = ["ReadWriteOnce"]

    def _generate_default_envs(self, is_daemon, resource_updated):
        """
        Add essential variables to all system containers
        :param is_daemon:
        :return:
        """
        default_envs = [
            # Kubernetes downward APIs
            {
                "name": "AX_NODE_NAME",
                "path": "spec.nodeName"
            },
            {
                "name": "AX_POD_NAME",
                "path": "metadata.name"
            },
            {
                "name": "AX_POD_NAMESPACE",
                "path": "metadata.namespace"
            },
            {
                "name": "AX_POD_IP",
                "path": "status.podIP"
            },

            # Values
            {
                "name": "DISK_MULT",
                "value": str(self.disk_mult)
            },
            {
                "name": "AX_TARGET_CLOUD",
                "value": Cloud().target_cloud()
            },
            {
                "name": "AX_CLUSTER_NAME_ID",
                "value": self._cluster_name_id
            },
            {
                "name": "AX_CUSTOMER_ID",
                "value": AXCustomerId().get_customer_id()
            },
        ]

        # Special cases for daemons
        if is_daemon:
            if resource_updated:
                default_envs += [
                    {
                        "name": "CPU_MULT",
                        "value": str(self.daemon_cpu_mult)
                    },
                    {
                        "name": "MEM_MULT",
                        "value": str(self.daemon_mem_mult)
                    },
                ]
            else:
                default_envs += [
                    {
                        "name": "CPU_MULT",
                        "value": "1.0"
                    },
                    {
                        "name": "MEM_MULT",
                        "value": "1.0"
                    },
                ]
        else:
            default_envs += [
                {
                    "name": "CPU_MULT",
                    "value": str(self.cpu_mult)
                },
                {
                    "name": "MEM_MULT",
                    "value": str(self.mem_mult)
                },
            ]

        rst = []
        for d in default_envs:
            var = V1EnvVar()
            var.name = d["name"]

            if d.get("path", None):
                field = V1ObjectFieldSelector()
                field.field_path = d["path"]
                src = V1EnvVarSource()
                src.field_ref = field
                var.value_from = src
            else:
                var.value = d["value"]
            rst.append(var)
        return rst