class Application(object): """ Create an Application which maps to a kubernetes namespace """ def __init__(self, name, client=None): self.name = name if client is None: self._client = KubernetesApiClient(use_proxy=True) else: self._client = client self._registry_spec = None self._software_info = SoftwareInfo() if self._software_info.registry_is_private(): secret = KubeObjectConfigFile( DEFAULT_SECRET_YAML_PATH, {"REGISTRY_SECRETS": self._software_info.registry_secrets}) for obj in secret.get_swagger_objects(): if isinstance(obj, swagger_client.V1Secret): self._registry_spec = obj assert self._registry_spec, "Argo registry specification is missing" self._am_service_spec = None self._am_deployment_spec = None # AA-2471: Hack to add AXOPS_EXT_DNS to Application Manager elb = InternalRoute("axops", "axsys", client=self._client) elb_status = elb.status(with_loadbalancer_info=True)["loadbalancer"][0] if not elb_status: raise AXPlatformException( "Could not get axops elb address {}".format(elb_status)) replacements = { "NAMESPACE": self._software_info.image_namespace, "VERSION": self._software_info.image_version, "REGISTRY": self._software_info.registry, "APPLICATION_NAME": self.name, "AXOPS_EXT_DNS": elb_status } cluster_name_id = os.getenv("AX_CLUSTER_NAME_ID", None) assert cluster_name_id, "Cluster name id is None!" cluster_config = AXClusterConfig(cluster_name_id=cluster_name_id) if cluster_config.get_cluster_provider() != ClusterProvider.USER: axam_path = DEFAULT_AM_YAML_PATH else: axam_path = "/ax/config/service/argo-all/axam-svc.yml.in" replacements["ARGO_DATA_BUCKET_NAME"] = os.getenv( "ARGO_DATA_BUCKET_NAME") logger.info("Using replacements: %s", replacements) k = KubeObjectConfigFile(axam_path, replacements) for obj in k.get_swagger_objects(): if isinstance(obj, swagger_client.V1Service): self._am_service_spec = obj elif isinstance(obj, swagger_client.V1beta1Deployment): self._am_deployment_spec = obj self._add_pod_metadata("deployment", self._am_deployment_spec.metadata.name, is_label=True) self._add_pod_metadata( "ax_costid", json.dumps({ "app": self.name, "service": "axam-deployment", "user": "******" })) else: logger.debug("Ignoring specification of type {}".format( type(obj))) assert self._am_service_spec and self._am_deployment_spec, "Application monitor specification is missing" def _add_pod_metadata(self, key, value, is_label=False): """ Helper function to add metadata to deployment pod spec for AXAM """ pod_meta = self._am_deployment_spec.spec.template.metadata if is_label: if pod_meta.labels is None: pod_meta.labels = {} pod_meta.labels[key] = value else: if pod_meta.annotations is None: pod_meta.annotations = {} pod_meta.annotations[key] = value def create(self, force_recreate=False): """ Create a kubernetes namespace and populate it with argo registry Idempotency: This function will be idempotent as long as the content of the secret is not changed. If create is called with a registry secret that has been updated and the namespace with the secret already exists then it will not update the secret for now. """ @retry_not_exists def create_ns_in_provider(): namespace = swagger_client.V1Namespace() namespace.metadata = swagger_client.V1ObjectMeta() namespace.metadata.name = self.name self._client.api.create_namespace(namespace) # NOTE: 403 is not retried as application is getting deleted in parallel # 422 is unprocessable object (aka error in spec) @retry_unless(status_code=[403, 422]) def create_reg_in_provider(): if self._registry_spec is None: return try: self._client.api.create_namespaced_secret( self._registry_spec, self.name) except swagger_client.rest.ApiException as e: if e.status == 409: self._client.api.patch_namespaced_secret( self._registry_spec.to_dict(), self.name, self._registry_spec.metadata.name) else: raise e @retry_unless(status_code=[403, 422]) def create_app_monitor_service_in_provider(): try: self._client.api.create_namespaced_service( self._am_service_spec, self.name) except swagger_client.rest.ApiException as e: if e.status == 409: self._client.api.patch_namespaced_service( self._am_service_spec.to_dict(), self.name, self._am_service_spec.metadata.name) else: raise e @retry_unless(status_code=[403, 422]) def create_app_monitor_deployment_in_provider(): try: self._client.apisappsv1beta1_api.create_namespaced_deployment( self._am_deployment_spec, self.name) except swagger_client.rest.ApiException as e: if e.status == 409: if force_recreate: # add a new metadata in pod spec to force the recreation of pods self._add_pod_metadata( "applatix.io/force-recreate-salt", str(uuid.uuid4())) self._client.apisappsv1beta1_api.replace_namespaced_deployment( self._am_deployment_spec, self.name, self._am_deployment_spec.metadata.name) else: raise e try: logger.debug("Creating application {}".format(self.name)) create_ns_in_provider() logger.debug("Created namespace {}".format(self.name)) create_reg_in_provider() create_app_monitor_service_in_provider() logger.debug("Created application monitor service {}".format( self._am_service_spec.metadata.name)) create_app_monitor_deployment_in_provider() logger.debug("Created application monitor deployment {}".format( self._am_deployment_spec.metadata.name)) except Exception as e: logger.exception(e) def delete(self, timeout=None): """ Delete a kubernetes namespace and image secret for Argo Idempotency: Can be repeatedly called """ delete_grace_period = 1 options = swagger_client.V1DeleteOptions() options.grace_period_seconds = delete_grace_period options.orphan_dependents = False @retry_unless(swallow_code=[404, 409]) def delete_ns_in_provider(): """ The retry is not done for 404 (not found) and also for 409 (conflict) The 404 case is for simple retry. 409 happens when application delete was requested but not complete and another request came in. """ logger.debug("Deleting application {}".format(self.name)) self._client.api.delete_namespace(options, self.name) delete_ns_in_provider() start_time = time.time() while self.exists(): logger.debug("Application {} still exists".format(self.name)) time.sleep(delete_grace_period + 1) wait_time = int(time.time() - start_time) if timeout is not None and wait_time > timeout: raise AXTimeoutException( "Could not delete namespace {} in {} seconds".format( self.name, timeout)) def exists(self): @retry_unless_not_found def get_ns_in_provider(): try: stat = self._client.api.read_namespace(self.name) return True except swagger_client.rest.ApiException as e: if e.status == 404: return False else: raise e return get_ns_in_provider() def status(self): """ This function checks the following: 1. Namespace exists? 2. Argo Registry exists? 3. TODO: Application Monitor exists Returns: A json dict with the status of each { 'namespace': True/False, 'registry': True/False, 'monitor': True/False } """ ret = {'namespace': False, 'registry': False, 'monitor': False} if not self.exists(): return ret ret['namespace'] = True ns = self._get_registry_from_provider() if ns is None: return ret ret['registry'] = True srv = self._get_am_service_from_provider() if srv is None: return ret am_dep = self._get_am_deployment_from_provider() if am_dep is not None and am_dep.status.available_replicas == am_dep.status.replicas: ret["monitor"] = True return ret def healthy(self): """ If all components are present/healthy then return True else return False """ d = self.status() for component in d: if not d[component]: return False return True def events(self, name=None): return self._get_events_from_provider(name).items @retry_unless(swallow_code=[404]) def _get_registry_from_provider(self): if self._registry_spec is not None: return self._client.api.read_namespaced_secret( self.name, self._registry_spec.metadata.name) else: return "NotNeeded" @retry_unless(swallow_code=[404]) def _get_am_service_from_provider(self): return self._client.api.read_namespaced_service( self.name, self._am_service_spec.metadata.name) @retry_unless(swallow_code=[404]) def _get_am_deployment_from_provider(self): return self._client.apisappsv1beta1_api.read_namespaced_deployment( self.name, self._am_deployment_spec.metadata.name) @retry_unless(swallow_code=[404]) def _get_events_from_provider(self, name): # XXX: For some reason list_namespaced_event does not take a namespace but the _21 version # of the function does. Hopefully this gets fixed in swagger soon field_selector = None if name is not None: field_selector = "involvedObject.name={}".format(name) return self._client.api.list_namespaced_event( self.name, field_selector=field_selector)
class Container(KubeObject): """ Class for creating container specifications """ LIVENESS_PROBE = 1 READINESS_PROBE = 2 def __init__(self, name, image, pull_policy=None): """ Construct a container that will provide the spec for a kubernetes container http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_container Args: name: name of a container. must be conformant to kubernetes container name image: image for container pull_policy: pull policy based on kubernetes. If None then kubernetes default is used """ self.name = name self.image = image self.image_pull_policy = pull_policy self.command = None self.args = None self.vmap = {} self.env_map = {} self.ports = [] self.resources = None self.privileged = None self.software_info = SoftwareInfo() self.probes = {} def generate_spec(self): c = swagger_client.V1Container() c.name = self.name c.image = self.image if self.resources is not None: c.resources = swagger_client.V1ResourceRequirements() c.resources.requests = {} c.resources.limits = {} if "cpu_cores" in self.resources: c.resources.requests["cpu"] = str( self.resources["cpu_cores"][0]) if self.resources["cpu_cores"][1] is not None: c.resources.limits["cpu"] = str( self.resources["cpu_cores"][1]) if "mem_mib" in self.resources: c.resources.requests["memory"] = "{}Mi".format( self.resources["mem_mib"][0]) if self.resources["mem_mib"][1] is not None: c.resources.limits["memory"] = "{}Mi".format( self.resources["mem_mib"][1]) # Kubernetes 1.5 requires init container must specify image pull policy. Since we are setting # a pull policy for all containers, we want to replicate the kubernetes default behavior of pulling # the image if tag is "latest" if self.image.endswith(':latest'): c.image_pull_policy = ContainerImagePullPolicy.PullAlways else: c.image_pull_policy = self.image_pull_policy or ContainerImagePullPolicy.PullIfNotPresent if self.command: c.command = self.command if self.args: c.args = self.args c.volume_mounts = [] for _, vol in self.vmap.iteritems(): c.volume_mounts.append(vol.get_container_spec()) c.env = [] for _, env in self.env_map.iteritems(): c.env.append(env) if self.privileged is not None: c.security_context = swagger_client.V1SecurityContext() c.security_context.privileged = self.privileged for probe in self.probes: probe_spec = self.probes[probe] probe_k8s_spec = Container._generate_probe_spec(probe_spec) if probe == Container.LIVENESS_PROBE: c.liveness_probe = probe_k8s_spec elif probe == Container.READINESS_PROBE: c.readiness_probe = probe_k8s_spec else: raise AXIllegalArgumentException( "Unexpected probe type {} found with spec {}".format( probe, probe_spec)) return c def add_resource_constraints(self, resource, request, limit=None): if self.resources is None: self.resources = {} self.resources[resource] = (request, limit) def add_volume(self, volume): self.vmap[volume.name] = volume def add_volumes(self, volumes): for vol in volumes or []: self.add_volume(vol) def get_volume(self, name): return self.vmap.get(name, None) def add_env(self, name, value=None, value_from=None): env = swagger_client.V1EnvVar() env.name = name if value is not None: env.value = value else: assert value_from is not None, "value and value_from both cannot be None for env {}".format( name) env.value_from = swagger_client.V1EnvVarSource() env.value_from.field_ref = swagger_client.V1ObjectFieldSelector() env.value_from.field_ref.field_path = value_from # Some 1.5 requires this. https://github.com/kubernetes/kubernetes/issues/39189 env.value_from.field_ref.api_version = "v1" self.env_map[name] = env def add_probe(self, probe_type, probe_spec): self.probes[probe_type] = probe_spec def parse_probe_spec(self, container_template): """ @type container_template: argo.template.v1.container.ContainerTemplate """ if container_template.liveness_probe: probe_type = Container.LIVENESS_PROBE self.add_probe(probe_type, container_template.liveness_probe) if container_template.readiness_probe: probe_type = Container.READINESS_PROBE self.add_probe(probe_type, container_template.readiness_probe) def get_registry(self, namespace="axuser"): """ This function returns the name of the secrets file that needs to be used in the pod specification image_pull_secrets array """ (reg, _, _) = DockerImage(fullname=self.image).docker_names() if reg == self.software_info.registry: if self.software_info.registry_is_private(): return "applatix-registry" else: return None else: try: smanager = SecretsManager() secret = smanager.get_imgpull(reg, namespace) if secret: return secret.metadata.name # Code for copying the registry to the app namespace if # it does not exist. We do not copy to axuser as secrets # are always created there. secret_axuser = smanager.get_imgpull(reg, "axuser") if secret_axuser and namespace != "axuser": smanager.copy_imgpull(secret_axuser, namespace) return secret_axuser.metadata.name except Exception as e: logger.debug( "Did not find a secret for registry {} due to exception {}" .format(reg, e)) return None def volume_iterator(self): for _, vol in self.vmap.iteritems(): yield vol @staticmethod def _generate_probe_spec(spec): """ @type spec argo.template.v1.container.ContainerProbe """ try: probe = swagger_client.V1Probe() probe.initial_delay_seconds = spec.initial_delay_seconds probe.timeout_seconds = spec.timeout_seconds probe.period_seconds = spec.period_seconds probe.failure_threshold = spec.failure_threshold probe.success_threshold = spec.success_threshold if spec.exec_probe: action = swagger_client.V1ExecAction() action.command = shlex.split(spec.exec_probe.command) probe._exec = action return probe elif spec.http_get: action = swagger_client.V1HTTPGetAction() action.path = spec.http_get.path action.port = spec.http_get.port headers = spec.http_get.http_headers action.http_headers = [] for header in headers or []: h = swagger_client.V1HTTPHeader() h.name = header["name"] h.value = header["value"] action.http_headers.append(h) probe.http_get = action return probe else: logger.debug("Cannot handle probe {}".format(spec)) except Exception as e: raise AXIllegalArgumentException( "Probe {} cannot be processed due to error {}".format(spec, e)) return None
class AXSYSKubeYamlUpdater(object): """ This class loads a kubernetes yaml file, updates resource, and generate objects that kube_object.py can consume """ def __init__(self, config_file_path): assert os.path.isfile( config_file_path), "Config file {} is not a file".format( config_file_path) self._config_file = config_file_path self._cluster_name_id = AXClusterId().get_cluster_name_id() self._cluster_config = AXClusterConfig( cluster_name_id=self._cluster_name_id) self.cpu_mult, self.mem_mult, self.disk_mult, \ self.daemon_cpu_mult, self.daemon_mem_mult = self._get_resource_multipliers() self._swagger_components = [] self._yaml_components = [] self._updated_raw = "" # TODO: when we support config software info using a config file, need to figure out how that # file gets passed through, since SoftwareInfo is not a singleton self._software_info = SoftwareInfo() self._load_objects() self._load_raw() @property def updated_raw(self): return self._updated_raw @property def components_in_dict(self): return self._yaml_components @property def components_in_swagger(self): return self._swagger_components def _load_objects(self): with open(self._config_file, "r") as f: data = f.read() for c in yaml.load_all(data): swagger_obj = self._config_yaml(c) yaml_obj = ApiClient().sanitize_for_serialization(swagger_obj) self._swagger_components.append(swagger_obj) self._yaml_components.append(yaml_obj) def _load_raw(self): self._updated_raw = yaml.dump_all(self._yaml_components) def _get_resource_multipliers(self): """ Resources in yaml templates need to be multiplied with these numbers :return: cpu_multiplier, mem_multiplier, disk_multiplier """ # Getting cluster size from cluster config, in order to configure resources # There are 3 situations we will be using AXClusterConfig # - During install, since the class is a singleton, it has all the values we need # no need to download from s3 # - During upgrade, since we are exporting AWS_DEFAULT_PROFILE, we can download # cluster config files from s3 to get the values # - During job creation: the node axmon runs has the proper roles to access s3 try: ax_node_max = int(self._cluster_config.get_asxys_node_count()) ax_node_type = self._cluster_config.get_axsys_node_type() usr_node_max = int( self._cluster_config.get_max_node_count()) - ax_node_max usr_node_type = self._cluster_config.get_axuser_node_type() assert all( [ax_node_max, ax_node_type, usr_node_max, usr_node_type]) except Exception as e: logger.error( "Unable to read cluster config, skip resource config for %s. Error %s", self._config_file, e) return 1, 1, 1, 1, 1 rc = AXSYSResourceConfig( ax_node_type=ax_node_type, ax_node_max=ax_node_max, usr_node_type=usr_node_type, usr_node_max=usr_node_max, cluster_type=self._cluster_config.get_ax_cluster_type()) #logger.info("With %s %s axsys nodes, %s %s axuser nodes, component %s uses multipliers (%s, %s, %s, %s, %s)", # ax_node_max, ax_node_type, usr_node_max, usr_node_type, self._config_file, # rc.cpu_multiplier, rc.mem_multiplier, rc.disk_multiplier, # rc.daemon_cpu_multiplier, rc.daemon_mem_multiplier) return rc.cpu_multiplier, rc.mem_multiplier, rc.disk_multiplier, rc.daemon_cpu_multiplier, rc.daemon_mem_multiplier def _config_yaml(self, kube_yaml_obj): """ Load dict into swagger object, patch resource, sanitize, return a dict :param kube_yaml_obj: :return: swagger object with resource values finalized """ kube_kind = kube_yaml_obj["kind"] (swagger_class_literal, swagger_instance) = KubeKindToV1KubeSwaggerObject[kube_kind] swagger_obj = ApiClient()._ApiClient__deserialize( kube_yaml_obj, swagger_class_literal) assert isinstance(swagger_obj, swagger_instance), \ "{} has instance {}, expected {}".format(swagger_obj, type(swagger_obj), swagger_instance) if isinstance(swagger_obj, V1beta1Deployment): if not self._software_info.registry_is_private(): swagger_obj.spec.template.spec.image_pull_secrets = None node_selector = swagger_obj.spec.template.spec.node_selector if node_selector.get('ax.tier', 'applatix') == 'master': # Skip updating containers on master. logger.info( "Skip updating cpu, mem multipliers for pods on master: %s", swagger_obj.metadata.name) else: for container in swagger_obj.spec.template.spec.containers: self._update_container(container) return swagger_obj elif isinstance(swagger_obj, V1Pod): if not self._software_info.registry_is_private(): swagger_obj.spec.image_pull_secrets = None return swagger_obj elif isinstance(swagger_obj, V1beta1DaemonSet): if not self._software_info.registry_is_private(): swagger_obj.spec.template.spec.image_pull_secrets = None for container in swagger_obj.spec.template.spec.containers: # We are special-casing applet DaemonSet to compromise the fact that # we are using different node type for compute-intense nodes if swagger_obj.metadata.name == "applet": self._update_container(container=container, is_daemon=True, update_resource=True) else: self._update_container(container=container, is_daemon=True, update_resource=False) return swagger_obj elif isinstance(swagger_obj, V1beta1StatefulSet): if not self._software_info.registry_is_private(): swagger_obj.spec.template.spec.image_pull_secrets = None return self._update_statefulset(swagger_obj) elif isinstance(swagger_obj, V1PersistentVolumeClaim): self._update_volume(swagger_obj) return swagger_obj else: # logger.info("Object %s does not need to configure resource", type(swagger_obj)) # HACK, as the original hook will be messed up if isinstance(swagger_obj, V1Service): if swagger_obj.metadata.name == "axops": swagger_obj.spec.load_balancer_source_ranges = [] for cidr in self._cluster_config.get_trusted_cidr(): # Seems swagger client does not support unicode ... SIGH swagger_obj.spec.load_balancer_source_ranges.append( str(cidr)) # HACK #2: if we don't do this, kubectl will complain about something such as # # spec.ports[0].targetPort: Invalid value: "81": must contain at least one letter (a-z) # # p.target_port is defined as string though, but if its really a string, kubectl # is looking for a port name, rather than a number # SIGH ... for p in swagger_obj.spec.ports or []: try: p.target_port = int(p.target_port) except (ValueError, TypeError): pass return swagger_obj def _update_deployment_or_daemonset(self, kube_obj): assert isinstance(kube_obj, V1beta1Deployment) or isinstance( kube_obj, V1beta1DaemonSet) for container in kube_obj.spec.template.spec.containers: self._update_container(container) return kube_obj def _update_statefulset(self, kube_obj): assert isinstance(kube_obj, V1beta1StatefulSet) for container in kube_obj.spec.template.spec.containers: self._update_container(container) if isinstance(kube_obj.spec.volume_claim_templates, list): for vol in kube_obj.spec.volume_claim_templates: self._update_volume(vol) return kube_obj def _update_container(self, container, is_daemon=False, update_resource=True): assert isinstance(container, V1Container) if update_resource: cpulim = container.resources.limits.get("cpu") memlim = container.resources.limits.get("memory") cpureq = container.resources.requests.get("cpu") memreq = container.resources.requests.get("memory") def _massage_cpu(orig): return orig * self.daemon_cpu_mult if is_daemon else orig * self.cpu_mult def _massage_mem(orig): return orig * self.daemon_mem_mult if is_daemon else orig * self.mem_mult if cpulim: rvc = ResourceValueConverter(value=cpulim, target="cpu") rvc.massage(_massage_cpu) container.resources.limits["cpu"] = "{}m".format( rvc.convert("m")) if cpureq: rvc = ResourceValueConverter(value=cpureq, target="cpu") rvc.massage(_massage_cpu) container.resources.requests["cpu"] = "{}m".format( rvc.convert("m")) if memlim: rvc = ResourceValueConverter(value=memlim, target="memory") rvc.massage(_massage_mem) container.resources.limits["memory"] = "{}Mi".format( int(rvc.convert("Mi"))) if memreq: rvc = ResourceValueConverter(value=memreq, target="memory") rvc.massage(_massage_mem) container.resources.requests["memory"] = "{}Mi".format( int(rvc.convert("Mi"))) if container.liveness_probe and container.liveness_probe.http_get: try: container.liveness_probe.http_get.port = int( container.liveness_probe.http_get.port) except (ValueError, TypeError): pass if container.readiness_probe and container.readiness_probe.http_get: try: container.readiness_probe.http_get.port = int( container.readiness_probe.http_get.port) except (ValueError, TypeError): pass # Add resource multiplier to containers in case we need them if not container.env: container.env = [] container.env += self._generate_default_envs(is_daemon, update_resource) def _update_volume(self, vol): assert isinstance(vol, V1PersistentVolumeClaim) vol_size = vol.spec.resources.requests["storage"] def _massage_disk(orig): return orig * self.disk_mult if vol_size: rvc = ResourceValueConverter(value=vol_size, target="storage") rvc.massage(_massage_disk) # Since AWS does not support value such as 1.5G, lets round up to its ceil vol.spec.resources.requests["storage"] = "{}Gi".format( int(ceil(rvc.convert("Gi")))) # Manually patch access mode as swagger client mistakenly interprets this as map vol.spec.access_modes = ["ReadWriteOnce"] def _generate_default_envs(self, is_daemon, resource_updated): """ Add essential variables to all system containers :param is_daemon: :return: """ default_envs = [ # Kubernetes downward APIs { "name": "AX_NODE_NAME", "path": "spec.nodeName" }, { "name": "AX_POD_NAME", "path": "metadata.name" }, { "name": "AX_POD_NAMESPACE", "path": "metadata.namespace" }, { "name": "AX_POD_IP", "path": "status.podIP" }, # Values { "name": "DISK_MULT", "value": str(self.disk_mult) }, { "name": "AX_TARGET_CLOUD", "value": Cloud().target_cloud() }, { "name": "AX_CLUSTER_NAME_ID", "value": self._cluster_name_id }, { "name": "AX_CUSTOMER_ID", "value": AXCustomerId().get_customer_id() }, ] # Special cases for daemons if is_daemon: if resource_updated: default_envs += [ { "name": "CPU_MULT", "value": str(self.daemon_cpu_mult) }, { "name": "MEM_MULT", "value": str(self.daemon_mem_mult) }, ] else: default_envs += [ { "name": "CPU_MULT", "value": "1.0" }, { "name": "MEM_MULT", "value": "1.0" }, ] else: default_envs += [ { "name": "CPU_MULT", "value": str(self.cpu_mult) }, { "name": "MEM_MULT", "value": str(self.mem_mult) }, ] rst = [] for d in default_envs: var = V1EnvVar() var.name = d["name"] if d.get("path", None): field = V1ObjectFieldSelector() field.field_path = d["path"] src = V1EnvVarSource() src.field_ref = field var.value_from = src else: var.value = d["value"] rst.append(var) return rst