class AXPlatform(object): def __new__(cls, *args, **kwargs): if Cloud().target_cloud_gcp(): from .gke_platform import AXGKEPlatform return super(AXPlatform, cls).__new__(AXGKEPlatform) else: return super(AXPlatform, cls).__new__(cls) def __init__( self, cluster_name_id=None, aws_profile=None, debug=True, manifest_root=AXPlatformConfigDefaults.DefaultManifestRoot, config_file=AXPlatformConfigDefaults.DefaultPlatformConfigFile, software_info=None): """ AX Platform bootstrap :param cluster_name_id: cluster name id :param aws_profile: aws profile to authenticate all aws clients :param debug: debug mode :param manifest_root: root directory to all ax service objects """ self._software_info = software_info if software_info else SoftwareInfo( ) assert isinstance( self._software_info, SoftwareInfo ), "Wrong type ({}) of software info passed in.".format( self._software_info) self._aws_profile = aws_profile self._manifest_root = manifest_root self._config = AXPlatformConfig(config_file) logger.info("Using Kubernetes manifest from %s", self._manifest_root) logger.info("Using platform configuration \"%s\" from %s", self._config.name, config_file) self._cluster_name_id = AXClusterId( cluster_name_id).get_cluster_name_id() self._cluster_config = AXClusterConfig( cluster_name_id=self._cluster_name_id, aws_profile=self._aws_profile) self._cluster_config_path = AXClusterConfigPath(cluster_name_id) self._cluster_info = AXClusterInfo(self._cluster_name_id, aws_profile=self._aws_profile) self._region = self._cluster_config.get_region() if Cloud().target_cloud_aws(): self._account = AWSAccountInfo( aws_profile=self._aws_profile).get_account_id() else: self._account = "" self._bucket_name = self._cluster_config_path.bucket() self._bucket = Cloud().get_bucket(self._bucket_name, aws_profile=self._aws_profile, region=self._region) # In debug mode, when we failed to create an object, we don't delete it but just # leave it for debug. self._debug = debug # DNS self.cluster_dns_name = None # Get kube cluster config. Automatic if in pod already. self._kube_config = self._cluster_info.get_kube_config_file_path( ) if self._cluster_name_id else None if self._cluster_name_id: if not os.path.isfile(self._kube_config): logger.info( "Can't find config file at %s; downloading from s3", self._kube_config) self._kube_config = self._cluster_info.download_kube_config() assert os.path.isfile( self._kube_config), "No kube_config file available" # Kubernetes related objects and macros self.kube_namespaces = [AXNameSpaces.AXSYS, AXNameSpaces.AXUSER] self.kube_axsys_namespace = AXNameSpaces.AXSYS self.kube_user_namespace = AXNameSpaces.AXUSER self.kubectl = KubernetesApiClient(config_file=self._kube_config) self.kube_poll = KubeObjPoll(kubectl=self.kubectl) self._monitor = AXKubeMonitor(kubectl=self.kubectl) self._monitor.reload_monitors(namespace=self.kube_axsys_namespace) self._monitor.start() # Kube Objects self._kube_objects = {} self._replacing = {} def _load_kube_objects_from_steps(self, steps): """ Extract kube objects from steps in config, and load them into memory :param steps: list :return: """ for object_group in steps: assert isinstance(object_group, AXPlatformObjectGroup) for obj in object_group.object_set: assert isinstance(obj, AXPlatformObject) name = obj.name filename = obj.manifest namespace = obj.namespace if name in self._kube_objects: raise ValueError("Duplicated object name {}".format(name)) kubeobj_conf_path = os.path.join(self._manifest_root, filename) self._kube_objects[name] = KubeObject( config_file=kubeobj_conf_path, kubepoll=self.kube_poll, replacing=None, kube_config=self._kube_config, kube_namespace=namespace) def _generate_replacing(self): # Platform code are running in python 2.7, and therefore for trusted cidr list, the str() method # will return something like [u'54.149.149.230/32', u'73.70.250.25/32', u'104.10.248.90/32'], and # this 'u' prefix cannot be surpressed. With this prefix, our macro replacing would create invalid # yaml files, and therefore we construct string manually here trusted_cidr = self._cluster_config.get_trusted_cidr() if isinstance(trusted_cidr, list): trusted_cidr_str = "[" for cidr in trusted_cidr: trusted_cidr_str += "\"{}\",".format(str(cidr)) trusted_cidr_str = trusted_cidr_str[:-1] trusted_cidr_str += "]" else: trusted_cidr_str = "[{}]".format(trusted_cidr) axsys_cpu = 0 axsys_mem = 0 daemon_cpu = 0 daemon_mem = 0 for name in self._kube_objects.keys(): cpu, mem, dcpu, dmem = self._kube_objects[name].resource_usage axsys_cpu += cpu axsys_mem += mem daemon_cpu += dcpu daemon_mem += dmem # kube-proxy (100m CPU and 100Mi memory. Note kube-proxy does not # have a memory request, but this is an approximation) daemon_cpu += 100 daemon_mem += 100 logger.info( "Resource Usages: axsys_cpu: %s milicores, axsys_mem: %s Mi, node_daemon_cpu: %s milicores, node_daemon_mem: %s Mi", axsys_cpu, axsys_mem, daemon_cpu, daemon_mem) axsys_node_count = int(self._cluster_config.get_asxys_node_count()) axuser_min_count = str( int(self._cluster_config.get_min_node_count()) - axsys_node_count) axuser_max_count = str( int(self._cluster_config.get_max_node_count()) - axsys_node_count) autoscaler_scan_interval = str( self._cluster_config.get_autoscaler_scan_interval()) usr_node_cpu_rsvp = float(daemon_cpu) / EC2_PARAMS[ self._cluster_config.get_axuser_node_type()]["cpu"] usr_node_mem_rsvp = float(daemon_mem) / EC2_PARAMS[ self._cluster_config.get_axuser_node_type()]["memory"] scale_down_util_thresh = round( max(usr_node_cpu_rsvp, usr_node_mem_rsvp), 3) + 0.001 logger.info("Setting node scale down utilization threshold to %s", scale_down_util_thresh) self._persist_node_resource_rsvp(daemon_cpu, daemon_mem) with open("/kubernetes/cluster/version.txt", "r") as f: cluster_install_version = f.read().strip() # Prepare autoscaler asg_manager = AXUserASGManager(self._cluster_name_id, self._region, self._aws_profile) asg = asg_manager.get_variable_asg() or asg_manager.get_spot_asg( ) or asg_manager.get_on_demand_asg() if not asg: raise AXPlatformException( "Failed to get autoscaling group for cluster {}".format( self._cluster_name_id)) asg_name = asg["AutoScalingGroupName"] if not asg_name: logger.error("Autoscaling group name not found for %s", self._cluster_name_id) raise AXPlatformException("Cannot find cluster autoscaling group") # Prepare minion-manager. spot_instances_option = self._cluster_config.get_spot_instances_option( ) minion_manager_asgs = "" if spot_instances_option == SpotInstanceOption.ALL_SPOT: for asg in asg_manager.get_all_asgs(): minion_manager_asgs = minion_manager_asgs + asg[ "AutoScalingGroupName"] + " " minion_manager_asgs = minion_manager_asgs[:-1] elif spot_instances_option == SpotInstanceOption.PARTIAL_SPOT: minion_manager_asgs = asg_manager.get_variable_asg( )["AutoScalingGroupName"] return { "REGISTRY": self._software_info.registry, "REGISTRY_SECRETS": self._software_info.registry_secrets, "NAMESPACE": self._software_info.image_namespace, "VERSION": self._software_info.image_version, "AX_CLUSTER_NAME_ID": self._cluster_name_id, "AX_AWS_REGION": self._region, "AX_AWS_ACCOUNT": self._account, "AX_CUSTOMER_ID": AXCustomerId().get_customer_id(), "TRUSTED_CIDR": trusted_cidr_str, "NEW_KUBE_SALT_SHA1": os.getenv("NEW_KUBE_SALT_SHA1") or " ", "NEW_KUBE_SERVER_SHA1": os.getenv("NEW_KUBE_SERVER_SHA1") or " ", "AX_KUBE_VERSION": os.getenv("AX_KUBE_VERSION"), "AX_CLUSTER_INSTALL_VERSION": cluster_install_version, "SANDBOX_ENABLED": str(self._cluster_config.get_sandbox_flag()), "ARGO_LOG_BUCKET_NAME": self._cluster_config.get_support_object_store_name(), "ASG_MIN": axuser_min_count, "ASG_MAX": axuser_max_count, "AUTOSCALER_SCAN_INTERVAL": autoscaler_scan_interval, "SCALE_DOWN_UTIL_THRESH": str(scale_down_util_thresh), "AX_CLUSTER_META_URL_V1": self._bucket.get_object_url_from_key( key=self._cluster_config_path.cluster_metadata()), "ASG_NAME": asg_name, "DNS_SERVER_IP": os.getenv("DNS_SERVER_IP", default_kube_up_env["DNS_SERVER_IP"]), "AX_ENABLE_SPOT_INSTANCES": str(spot_instances_option != SpotInstanceOption.NO_SPOT), "AX_SPOT_INSTANCE_ASGS": minion_manager_asgs, } def _persist_node_resource_rsvp(self, user_node_daemon_cpu, user_node_daemon_mem): self._cluster_config.set_user_node_resource_rsvp( cpu=user_node_daemon_cpu, mem=user_node_daemon_mem) self._cluster_config.save_config() def start(self): """ Bring up platform using "platform-start.cfg" configuration from manifest directory :return: """ # Generate kube-objects steps = self._config.steps self._load_kube_objects_from_steps(steps) self._replacing = self._generate_replacing() # TODO: remove component's dependencies to AXOPS_EXT_DNS env (#32) # At this moment, we MUST separate first step due to the above dependency assert len(steps) >= 2, "Should have at least 1 step to create axops" self.create_objects(steps[0]) self.create_objects(steps[1]) self.create_objects(steps[2]) # Prepare axops_eip self._set_ext_dns() logger.debug("Replacing ENVs: %s", self._replacing) info_bound = "=======================================================\n" img_namespace = "Image Namespace: {}\n".format( self._software_info.image_namespace) img_version = "Image Version: {}\n".format( self._software_info.image_version) start_info = "\n\n{}{}{}{}{}".format( info_bound, "Platform Up: Bringing up Argo services...\n", img_namespace, img_version, info_bound) logger.info(start_info) # Start rest of the objects for i in range(3, len(steps)): self.create_objects(steps[i]) # update application namespace logger.info("Updating application managers") for app in Applications(client=self.kubectl).list(): logger.info("--- updating {}".format(app)) a = Application(app, client=self.kubectl) a.create(force_recreate=True) logger.info("Done updating application managers") # Upload version information to target cluster self._update_version() logger.info("\n\n%sCluster %s is up. Cluster is available at %s%s\n", COLOR_GREEN, self._cluster_name_id, self.cluster_dns_name, COLOR_NORM) def stop(self): """ Bring down platform using "platform-stop.cfg" configuration from manifest directory :return: """ # Generate kube-objects (Does not need to generate replacing during platform down) # Stop order should be the reverse of start steps = self._config.steps steps.reverse() self._load_kube_objects_from_steps(steps) info_bound = "=======================================================\n" stop_info = "\n\n{}{}{}".format( info_bound, "Platform Down: Shutting down Argo services...\n", info_bound) logger.info(stop_info) # Bring down objects according to steps for i in range(len(steps)): object_group = steps[i] self.delete_objects(object_group) def stop_monitor(self): self._monitor.stop() def create_objects(self, objects): """ Start kubernetes objects based on records. Wait for all of them. :param objects: AXPlatformObjectGroup """ assert isinstance(objects, AXPlatformObjectGroup) if not self._should_create_group( policy=objects.policy, policy_predicate=objects.policy_predicate, consistency=objects.consistency): logger.debug( "Skipping object group (%s) creation based on policy (%s), policy predicate (%s), consistency (%s)", objects.name, objects.policy, objects.policy_predicate, objects.consistency) return logger.info("Create step: %s", objects.name) logger.info("Creating platform objects\n\n%s", self._generate_object_summary(objects.object_set)) pool = ThreadPool(len(objects.object_set)) async_results = {} for obj in objects.object_set: assert isinstance(obj, AXPlatformObject) name = obj.name namespace = obj.namespace async_results[name] = pool.apply_async( self.start_one, args=(name, ), kwds={"namespace": namespace}) pool.close() pool.join() report, failed = self._generate_report(async_results, "Create") logger.info(report) if failed: raise AXPlatformException("Failed to create platform objects.") def _should_create_group(self, policy, policy_predicate, consistency): """ Take AXPlatformObjectGroup policy, predicate and consistency and determine if this group should be created or not :param policy: :param policy_predicate: :param consistency: :return: """ # Since we are not using consistency, we should always create if not # explicitly told not to, i.e. if there is a PrivateRegistryOnly # We are just leaving the interface here that should create or not # need to be decided by policy, policy_predicate and consistency if policy_predicate == ObjectGroupPolicyPredicate.PrivateRegistryOnly and \ not self._software_info.registry_is_private(): return False return True def delete_objects(self, objects): """ Stop kubernetes objects based on records. Wait for all of them. :param objects: AXPlatformObjectGroup """ assert isinstance(objects, AXPlatformObjectGroup) if not self._should_delete_group( policy=objects.policy, policy_predicate=objects.policy_predicate): logger.debug( "Skipping object group (%s) deletion based on policy (%s), policy predicate (%s)", objects.name, objects.policy, objects.policy_predicate) return logger.info("Delete step: %s", objects.name) logger.info("Deleting platform objects\n\n%s.", self._generate_object_summary(objects.object_set)) pool = ThreadPool(len(objects.object_set)) async_results = {} for obj in objects.object_set: assert isinstance(obj, AXPlatformObject) name = obj.name namespace = obj.namespace async_results[name] = pool.apply_async( self.stop_one, args=(name, ), kwds={"namespace": namespace}) pool.close() pool.join() report, failed = self._generate_report(async_results, "Delete") logger.info(report) if failed: raise AXPlatformException("Failed to create platform objects.") def _should_delete_group(self, policy, policy_predicate): """ Take AXPlatformObjectGroup policy and determine if this group should be deleted or not. Consistency is not needed for deletion :param policy: :param policy_predicate: :return: """ if policy == ObjectGroupPolicy.CreateMany: return True return False def start_one(self, name, namespace=AXNameSpaces.AXSYS): time.sleep( random.randint(0, AXPlatformConfigDefaults.ObjectOperationJitter)) logger.info("Creating %s in namespace %s ...", name, namespace) start = time.time() kube_obj = self._kube_objects[name] # Update them as there are new updates in replacing in platform start kube_obj.namespace = namespace kube_obj.replacing = self._replacing assert isinstance(kube_obj, KubeObject) result = { "name": name, "code": [], "events": [], "failed": False, "duration": "" } if kube_obj.healthy(): result["code"] += [ "{:.25s}:{}".format(name, KubeObjStatusCode.OBJ_EXISTS) ] result["duration"] = str(round(time.time() - start, 2)) return result # Previous platform start might fail, and might result in some componenets created # but not healthy (i.e. in CrashLoopBackoff). In this case, we delete the existing # object and try to create a new one if kube_obj.exists(): logger.warning( "Object %s exists but not healthy. Deleting object for idempotency ...", name) self.stop_one(name, namespace) assert not kube_obj.exists( ), "Kubeobject {} already created but is not healthy. Not Expected".format( name) monitor_info = kube_obj.get_create_monitor_info() if monitor_info: # use monitor waiters = [] # Create and register waiters for all objects that can be monitored for m in monitor_info: wait_info = { "kind": KubeKindToKubeApiObjKind[m.kube_kind], "name": m.name, "validator": m.validator } waiter = KubeObjWaiter() waiters.append((waiter, wait_info)) AXKubeMonitor().wait_for_kube_object( wait_info, AXPlatformConfigDefaults.ObjCreateWaitTimeout, waiter) # Call kubectl create kube_obj.create() # Wait on all waiters to retrieve status and events for waiter, wait_info in waiters: waiter.wait() result["events"] += waiter.details result["code"].append("{:.25s}:{}".format( wait_info["name"], waiter.result)) if waiter.result == KubeObjStatusCode.OK or waiter.result == KubeObjStatusCode.WARN: logger.info("Successfully created %s with code %s.", wait_info["name"], waiter.result) else: result["failed"] = True logger.error( "Failed to create %s in %s with code %s. Events: %s", wait_info["name"], namespace, waiter.result, str(waiter.details)) if not self._debug: logger.info("Deleting %s due to creation failure", name) del_rst = self.stop_one(name, namespace) result["code"] += del_rst["code"] result["events"] += del_rst["events"] result["duration"] = str(round(time.time() - start, 2)) return result # Poll extra if required (for Petset and Deployments with multiple replicas) if kube_obj.extra_poll: logger.info( "Polling till healthy to make sure rest of components of %s are up and running ...", name) create_rst = self._poll_till_healthy( name=name, kube_obj=kube_obj, start_time=start, poll_interval=AXPlatformConfigDefaults. ObjCreateExtraPollInterval, poll_max_retry=AXPlatformConfigDefaults. ObjCreateExtraPollMaxRetry, rst=result) if create_rst["failed"] and not self._debug: logger.info("Deleting %s due to creation failure", name) del_rst = self.stop_one(name, namespace) create_rst["code"] += del_rst["code"] create_rst["events"] += del_rst["events"] create_rst["duration"] = str(round(time.time() - start, 2)) return create_rst # Poll once to confirm all components from this Kubernetes config file exist, # In case there are objects in this config file cannot be monitored, i.e. svc # without elb. This is really not expected so we don't delete it if not kube_obj.healthy(): logger.error( "Object %s created but is not healthy. This is NOT EXPECTED, please check manually.", name) result["code"].append("{:.25s}:{}".format( name, KubeObjStatusCode.UNHEALTHY)) result["failed"] = True result["events"].append( "Object {} created byt is not healthy".format(name)) result["duration"] = str(round(time.time() - start, 2)) if not result["failed"]: logger.info("Successfully created object %s.", name) return result else: # use polling kube_obj.create() create_rst = self._poll_till_healthy( name=name, kube_obj=kube_obj, start_time=start, poll_interval=AXPlatformConfigDefaults.ObjCreatePollInterval, poll_max_retry=AXPlatformConfigDefaults.ObjCreatePollMaxRetry, rst=result) if create_rst["failed"] and not self._debug: logger.info("Deleting %s due to creation failure", name) del_rst = self.stop_one(name, namespace) create_rst["code"] += del_rst["code"] create_rst["events"] += del_rst["events"] create_rst["duration"] = str(round(time.time() - start, 2)) return create_rst @staticmethod def _poll_till_healthy(name, kube_obj, start_time, poll_interval, poll_max_retry, rst): trail = 0 assert isinstance(kube_obj, KubeObject) while True: if not kube_obj.healthy(): trail += 1 if trail > poll_max_retry: logger.error("Failed to create KubeObject %s", name) rst["code"] += [ "{:.25s}:{}".format(name, KubeObjStatusCode.UNHEALTHY) ] rst["events"] += [ "Object {} creation timeout. Not healthy".format(name) ] rst["failed"] = True rst["duration"] = str(round(time.time() - start_time, 2)) return rst else: logger.info("Successfully created %s.", name) rst["code"] += [ "{:.25s}:{}".format(name, KubeObjStatusCode.OK) ] rst["failed"] = False rst["duration"] = str(round(time.time() - start_time, 2)) return rst time.sleep(poll_interval) def stop_one(self, name, namespace=AXNameSpaces.AXSYS): time.sleep( random.randint(0, AXPlatformConfigDefaults.ObjectOperationJitter)) logger.info("Deleting %s in namespace %s ...", name, namespace) start = time.time() kube_obj = self._kube_objects[name] kube_obj.namespace = namespace kube_obj.replacing = self._replacing assert isinstance(kube_obj, KubeObject) result = { "name": name, "code": [], "events": [], "failed": False, "duration": "" } # Don't delete if object does not exist if not kube_obj.exists(): result["code"] += [ "{:.25s}:{}".format(name, KubeObjStatusCode.DELETED) ] result["duration"] = str(round(time.time() - start, 2)) return result monitor_info = kube_obj.get_delete_monitor_info() if monitor_info: # use monitor waiters = [] # Create and register waiters for all objects that can be monitored for m in monitor_info: wait_info = { "kind": KubeKindToKubeApiObjKind[m.kube_kind], "name": m.name, "validator": m.validator } waiter = KubeObjWaiter() waiters.append((waiter, wait_info)) AXKubeMonitor().wait_for_kube_object( wait_info, AXPlatformConfigDefaults.ObjDeleteWaitTimeout, waiter) # Call kubectl delete kube_obj.delete() # Wait on all waiters to retrieve status and events for waiter, wait_info in waiters: waiter.wait() result["events"] += waiter.details if waiter.result == KubeObjStatusCode.OK or waiter.result == KubeObjStatusCode.WARN: result["code"].append("{:.25s}:{}".format( wait_info["name"], KubeObjStatusCode.DELETED)) logger.info("Successfully deleted %s in %s with code %s.", wait_info["name"], name, result["code"]) else: result["failed"] = True result["code"].append("{:.25s}:{}".format( wait_info["name"], KubeObjStatusCode.UNKNOWN)) logger.error( "Failed to delete %s in %s with code %s. Events: %s", wait_info["name"], name, result["code"], str(waiter.details)) # Poll once to confirm all components from this Kubenetes config file exist # In case there are objects in this config file cannot be monitored, i.e. svc without elb if kube_obj.exists(): logger.error("Object %s deleted but still exists", name) result["failed"] = True result["code"].append("{:.25s}:{}".format( name, KubeObjStatusCode.UNKNOWN)) result["events"].append( "Object {} deleted but still exists.".format(name)) result["duration"] = str(round(time.time() - start, 2)) logger.info("Successfully deleted %s.", name) return result else: # use polling kube_obj.delete() return self._poll_till_not_exists( name=name, kube_obj=kube_obj, start_time=start, poll_interval=AXPlatformConfigDefaults.ObjDeletePollInterval, poll_max_retry=AXPlatformConfigDefaults.ObjDeletePollMaxRetry, rst=result) @staticmethod def _poll_till_not_exists(name, kube_obj, start_time, poll_interval, poll_max_retry, rst): trail = 0 assert isinstance(kube_obj, KubeObject) while True: if kube_obj.exists(): trail += 1 if trail > poll_max_retry: logger.error("Failed to delete KubeObject %s", name) rst["code"] += [ "{:.25s}:{}".format(name, KubeObjStatusCode.UNKNOWN) ] rst["events"] += [ "Object {} deletion timeout. Please manually check remaining pods" .format(name) ] rst["failed"] = True rst["duration"] = str(round(time.time() - start_time, 2)) return rst else: logger.info("Successfully deleted %s.", name) rst["code"] += [ "{:.25s}:{}".format(name, KubeObjStatusCode.DELETED) ] rst["failed"] = False rst["duration"] = str(round(time.time() - start_time, 2)) return rst time.sleep(poll_interval) def _generate_object_summary(self, objects): """ :param objects: list of AXPlatformObject :return: """ report_title = "\n{:25s} | {:110s} | {:20s}\n".format( "NAME", "MANIFEST", "NAMESPACE") report_bar = "{}\n".format("-" * 174) content = "" for obj in objects: assert isinstance(obj, AXPlatformObject) name = obj.name filename = os.path.join(self._manifest_root, obj.manifest) namespace = obj.namespace content += "{:25s} | {:110s} | {:20s}\n".format( name, filename, namespace) return report_title + report_bar + content @staticmethod def _generate_report(results, operation): failed = False report_body = "" warnings = "\n======= WARNING EVENTS =======\n" for name in results.keys(): individual_report = "{:25s} | {:110s} | {:20s}\n" individual_warning = "{name}: {events}\n\n" try: result = results[name].get() if result["failed"]: failed = True code = result["code"][0] for c in result["code"][1:]: code += " / {}".format(c) individual_report = individual_report.format( name, code, result["duration"], 2) if len(result["events"]) > 0: warnings += individual_warning.format( name=name, events=str(result["events"])) except Exception as e: failed = True logger.exception(str(e)) individual_report = individual_report.format( name, "EXCEPTION", "UNKNOWN") warnings += individual_warning.format(name=name, events=str(e)) report_body += individual_report report_head = "\n\nPlatform {} {}. Report:\n".format( operation, "FAILED" if failed else "SUCCESSFULLY") report_title = "\n{:25s} | {:110s} | {:20s}\n".format( "NAME", "STATUS", "TIME (sec)") report_bar = "{}\n".format("-" * 174) return "{}{}{}{}{}{}".format( report_head, report_title, report_bar, report_body, warnings, "==============================\n"), failed def _get_eip_from_config_map(self): try: cmd = [ "kubectl", "get", "configmap", "cluster-dns-name", "-o", "yaml", "--namespace", self.kube_axsys_namespace, "--kubeconfig", self._kube_config ] out = subprocess.check_output(cmd) return [yaml.load(out)["data"]["cluster-external-dns-name"]] except Exception: logger.error("Failed to get cluster dns name from config map.") return None def _get_svc_eip(self, svclabel, namespace): svc = self.kube_poll.poll_kubernetes_sync(KubeKind.SERVICE, namespace, svclabel) assert len( svc.items) == 1, "Currently services should only have one ingress" rst = [] for ig in svc.items[0].status.load_balancer.ingress: if ig.hostname: rst.append(ig.hostname) if ig.ip: rst.append(ig.ip) return rst def _set_ext_dns(self): axops_eip = self._get_eip_from_config_map() or self._get_svc_eip( svclabel="app=axops", namespace=AXNameSpaces.AXSYS) if not axops_eip: logger.error( "Platform Start Failed: cannot find External IP for AXOPS") raise AXPlatformException("AXOPS elastic IP does not exist") self.cluster_dns_name = axops_eip[0] # Don't change format of this message. Portal parses this line to get cluster IP/DNS. logger.info( "\n\n%s>>>>> Starting Argo platform... cluster DNS: %s%s\n", COLOR_GREEN, self.cluster_dns_name, COLOR_NORM) self._replacing["AXOPS_EXT_DNS"] = self.cluster_dns_name def get_cluster_external_dns(self): if not self.cluster_dns_name: self._set_ext_dns() return self.cluster_dns_name def _set_autoscaling(self): # Prepare autoscaler asg_manager = AXUserASGManager(self._cluster_name_id, self._region, self._aws_profile) asg = asg_manager.get_variable_asg() or asg_manager.get_spot_asg( ) or asg_manager.get_on_demand_asg() if not asg: raise AXPlatformException( "Failed to get autoscaling group for cluster {}".format( self._cluster_name_id)) asg_name = asg["AutoScalingGroupName"] if asg_name is not None: self._replacing["ASG_NAME"] = asg_name else: logger.error("Autoscaling group name not found for %s", self._cluster_name_id) raise AXPlatformException("Cannot find cluster autoscaling group") # TODO (#157) Version should only be uploaded during install and upgrade time def _update_version(self): # Software info we get during install / upgrade does not contain ami id # need to persist it as well self._software_info.ami_id = self._cluster_config.get_ami_id() AXVersion(AXCustomerId().get_customer_id(), self._cluster_name_id, self._aws_profile).update(self._software_info.to_dict())
class AXSYSKubeYamlUpdater(object): """ This class loads a kubernetes yaml file, updates resource, and generate objects that kube_object.py can consume """ def __init__(self, config_file_path): assert os.path.isfile( config_file_path), "Config file {} is not a file".format( config_file_path) self._config_file = config_file_path self._cluster_name_id = AXClusterId().get_cluster_name_id() self._cluster_config = AXClusterConfig( cluster_name_id=self._cluster_name_id) self.cpu_mult, self.mem_mult, self.disk_mult, \ self.daemon_cpu_mult, self.daemon_mem_mult = self._get_resource_multipliers() self._swagger_components = [] self._yaml_components = [] self._updated_raw = "" # TODO: when we support config software info using a config file, need to figure out how that # file gets passed through, since SoftwareInfo is not a singleton self._software_info = SoftwareInfo() self._load_objects() self._load_raw() @property def updated_raw(self): return self._updated_raw @property def components_in_dict(self): return self._yaml_components @property def components_in_swagger(self): return self._swagger_components def _load_objects(self): with open(self._config_file, "r") as f: data = f.read() for c in yaml.load_all(data): swagger_obj = self._config_yaml(c) yaml_obj = ApiClient().sanitize_for_serialization(swagger_obj) self._swagger_components.append(swagger_obj) self._yaml_components.append(yaml_obj) def _load_raw(self): self._updated_raw = yaml.dump_all(self._yaml_components) def _get_resource_multipliers(self): """ Resources in yaml templates need to be multiplied with these numbers :return: cpu_multiplier, mem_multiplier, disk_multiplier """ # Getting cluster size from cluster config, in order to configure resources # There are 3 situations we will be using AXClusterConfig # - During install, since the class is a singleton, it has all the values we need # no need to download from s3 # - During upgrade, since we are exporting AWS_DEFAULT_PROFILE, we can download # cluster config files from s3 to get the values # - During job creation: the node axmon runs has the proper roles to access s3 try: ax_node_max = int(self._cluster_config.get_asxys_node_count()) ax_node_type = self._cluster_config.get_axsys_node_type() usr_node_max = int( self._cluster_config.get_max_node_count()) - ax_node_max usr_node_type = self._cluster_config.get_axuser_node_type() assert all( [ax_node_max, ax_node_type, usr_node_max, usr_node_type]) except Exception as e: logger.error( "Unable to read cluster config, skip resource config for %s. Error %s", self._config_file, e) return 1, 1, 1, 1, 1 rc = AXSYSResourceConfig( ax_node_type=ax_node_type, ax_node_max=ax_node_max, usr_node_type=usr_node_type, usr_node_max=usr_node_max, cluster_type=self._cluster_config.get_ax_cluster_type()) #logger.info("With %s %s axsys nodes, %s %s axuser nodes, component %s uses multipliers (%s, %s, %s, %s, %s)", # ax_node_max, ax_node_type, usr_node_max, usr_node_type, self._config_file, # rc.cpu_multiplier, rc.mem_multiplier, rc.disk_multiplier, # rc.daemon_cpu_multiplier, rc.daemon_mem_multiplier) return rc.cpu_multiplier, rc.mem_multiplier, rc.disk_multiplier, rc.daemon_cpu_multiplier, rc.daemon_mem_multiplier def _config_yaml(self, kube_yaml_obj): """ Load dict into swagger object, patch resource, sanitize, return a dict :param kube_yaml_obj: :return: swagger object with resource values finalized """ kube_kind = kube_yaml_obj["kind"] (swagger_class_literal, swagger_instance) = KubeKindToV1KubeSwaggerObject[kube_kind] swagger_obj = ApiClient()._ApiClient__deserialize( kube_yaml_obj, swagger_class_literal) assert isinstance(swagger_obj, swagger_instance), \ "{} has instance {}, expected {}".format(swagger_obj, type(swagger_obj), swagger_instance) if isinstance(swagger_obj, V1beta1Deployment): if not self._software_info.registry_is_private(): swagger_obj.spec.template.spec.image_pull_secrets = None node_selector = swagger_obj.spec.template.spec.node_selector if node_selector.get('ax.tier', 'applatix') == 'master': # Skip updating containers on master. logger.info( "Skip updating cpu, mem multipliers for pods on master: %s", swagger_obj.metadata.name) else: for container in swagger_obj.spec.template.spec.containers: self._update_container(container) return swagger_obj elif isinstance(swagger_obj, V1Pod): if not self._software_info.registry_is_private(): swagger_obj.spec.image_pull_secrets = None return swagger_obj elif isinstance(swagger_obj, V1beta1DaemonSet): if not self._software_info.registry_is_private(): swagger_obj.spec.template.spec.image_pull_secrets = None for container in swagger_obj.spec.template.spec.containers: # We are special-casing applet DaemonSet to compromise the fact that # we are using different node type for compute-intense nodes if swagger_obj.metadata.name == "applet": self._update_container(container=container, is_daemon=True, update_resource=True) else: self._update_container(container=container, is_daemon=True, update_resource=False) return swagger_obj elif isinstance(swagger_obj, V1beta1StatefulSet): if not self._software_info.registry_is_private(): swagger_obj.spec.template.spec.image_pull_secrets = None return self._update_statefulset(swagger_obj) elif isinstance(swagger_obj, V1PersistentVolumeClaim): self._update_volume(swagger_obj) return swagger_obj else: # logger.info("Object %s does not need to configure resource", type(swagger_obj)) # HACK, as the original hook will be messed up if isinstance(swagger_obj, V1Service): if swagger_obj.metadata.name == "axops": swagger_obj.spec.load_balancer_source_ranges = [] for cidr in self._cluster_config.get_trusted_cidr(): # Seems swagger client does not support unicode ... SIGH swagger_obj.spec.load_balancer_source_ranges.append( str(cidr)) # HACK #2: if we don't do this, kubectl will complain about something such as # # spec.ports[0].targetPort: Invalid value: "81": must contain at least one letter (a-z) # # p.target_port is defined as string though, but if its really a string, kubectl # is looking for a port name, rather than a number # SIGH ... for p in swagger_obj.spec.ports or []: try: p.target_port = int(p.target_port) except (ValueError, TypeError): pass return swagger_obj def _update_deployment_or_daemonset(self, kube_obj): assert isinstance(kube_obj, V1beta1Deployment) or isinstance( kube_obj, V1beta1DaemonSet) for container in kube_obj.spec.template.spec.containers: self._update_container(container) return kube_obj def _update_statefulset(self, kube_obj): assert isinstance(kube_obj, V1beta1StatefulSet) for container in kube_obj.spec.template.spec.containers: self._update_container(container) if isinstance(kube_obj.spec.volume_claim_templates, list): for vol in kube_obj.spec.volume_claim_templates: self._update_volume(vol) return kube_obj def _update_container(self, container, is_daemon=False, update_resource=True): assert isinstance(container, V1Container) if update_resource: cpulim = container.resources.limits.get("cpu") memlim = container.resources.limits.get("memory") cpureq = container.resources.requests.get("cpu") memreq = container.resources.requests.get("memory") def _massage_cpu(orig): return orig * self.daemon_cpu_mult if is_daemon else orig * self.cpu_mult def _massage_mem(orig): return orig * self.daemon_mem_mult if is_daemon else orig * self.mem_mult if cpulim: rvc = ResourceValueConverter(value=cpulim, target="cpu") rvc.massage(_massage_cpu) container.resources.limits["cpu"] = "{}m".format( rvc.convert("m")) if cpureq: rvc = ResourceValueConverter(value=cpureq, target="cpu") rvc.massage(_massage_cpu) container.resources.requests["cpu"] = "{}m".format( rvc.convert("m")) if memlim: rvc = ResourceValueConverter(value=memlim, target="memory") rvc.massage(_massage_mem) container.resources.limits["memory"] = "{}Mi".format( int(rvc.convert("Mi"))) if memreq: rvc = ResourceValueConverter(value=memreq, target="memory") rvc.massage(_massage_mem) container.resources.requests["memory"] = "{}Mi".format( int(rvc.convert("Mi"))) if container.liveness_probe and container.liveness_probe.http_get: try: container.liveness_probe.http_get.port = int( container.liveness_probe.http_get.port) except (ValueError, TypeError): pass if container.readiness_probe and container.readiness_probe.http_get: try: container.readiness_probe.http_get.port = int( container.readiness_probe.http_get.port) except (ValueError, TypeError): pass # Add resource multiplier to containers in case we need them if not container.env: container.env = [] container.env += self._generate_default_envs(is_daemon, update_resource) def _update_volume(self, vol): assert isinstance(vol, V1PersistentVolumeClaim) vol_size = vol.spec.resources.requests["storage"] def _massage_disk(orig): return orig * self.disk_mult if vol_size: rvc = ResourceValueConverter(value=vol_size, target="storage") rvc.massage(_massage_disk) # Since AWS does not support value such as 1.5G, lets round up to its ceil vol.spec.resources.requests["storage"] = "{}Gi".format( int(ceil(rvc.convert("Gi")))) # Manually patch access mode as swagger client mistakenly interprets this as map vol.spec.access_modes = ["ReadWriteOnce"] def _generate_default_envs(self, is_daemon, resource_updated): """ Add essential variables to all system containers :param is_daemon: :return: """ default_envs = [ # Kubernetes downward APIs { "name": "AX_NODE_NAME", "path": "spec.nodeName" }, { "name": "AX_POD_NAME", "path": "metadata.name" }, { "name": "AX_POD_NAMESPACE", "path": "metadata.namespace" }, { "name": "AX_POD_IP", "path": "status.podIP" }, # Values { "name": "DISK_MULT", "value": str(self.disk_mult) }, { "name": "AX_TARGET_CLOUD", "value": Cloud().target_cloud() }, { "name": "AX_CLUSTER_NAME_ID", "value": self._cluster_name_id }, { "name": "AX_CUSTOMER_ID", "value": AXCustomerId().get_customer_id() }, ] # Special cases for daemons if is_daemon: if resource_updated: default_envs += [ { "name": "CPU_MULT", "value": str(self.daemon_cpu_mult) }, { "name": "MEM_MULT", "value": str(self.daemon_mem_mult) }, ] else: default_envs += [ { "name": "CPU_MULT", "value": "1.0" }, { "name": "MEM_MULT", "value": "1.0" }, ] else: default_envs += [ { "name": "CPU_MULT", "value": str(self.cpu_mult) }, { "name": "MEM_MULT", "value": str(self.mem_mult) }, ] rst = [] for d in default_envs: var = V1EnvVar() var.name = d["name"] if d.get("path", None): field = V1ObjectFieldSelector() field.field_path = d["path"] src = V1EnvVarSource() src.field_ref = field var.value_from = src else: var.value = d["value"] rst.append(var) return rst