def test_killpod(): k = KubernetesCluster() k.load(open(NODE1).read()) # trim resource, run only one Node k.load(open(PODS).read()) k.load(open(SERVICES).read()) k.load(open(REPLICASETS).read()) k.load(open(PRIORITYCLASSES).read()) k.load(open(DEPLOYMENT).read()) k.create_resource(open(DEPLOYMENT_NEW).read()) k._build_state() p = OptimisticRun( k.state_objects ) # TODO check me, i'd like to run exiction test with killpod execution # print_objects(k.state_objects) p.run(timeout=6600, sessionName="test_start_pods") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml()) assert "StartPod" in p.plan.__str__( ) # test for transition from Pending to Running #get pods only in Running state to check atNode value runningPods = filter( lambda z: z.status != STATUS_POD["Running"], (filter(lambda x: isinstance(x, mpod.Pod), k.state_objects))) nodes = filter(lambda x: isinstance(x, Node), k.state_objects) for pod in runningPods: assert pod.atNode in nodes._get_value( ) # check each pod than each have atNode killingPods = filter( lambda z: z.status != STATUS_POD["Killing"], (filter(lambda x: isinstance(x, mpod.Pod), k.state_objects))) assert len(killingPods) > 0 # test that some pod Killed
def test_eviction_fromfiles_strictgoal(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DAEMONET).read()) k._build_state() p = SingleGoalEvictionDetect(k.state_objects) p.select_target_service() p.run(timeout=6600, sessionName="test_eviction_fromfiles_strictgoal") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml()) if p.plan: i = 0 for a in p.plan: i = i + 1 print( i, ":", a.__class__.__name__, "\n", yaml.dump( { str(k): repr(v._get_value()) if v else f"NONE_VALUE:{v}" for (k, v) in a.kwargs.items() }, default_flow_style=False)) assert "StartPod" in p.plan.__str__()
def test_load_deployments(): k = KubernetesCluster() k.load(open(TEST_PRIORITYCLASS).read()) k.load(open(TEST_NODES).read()) k.load(open(TEST_PODS).read()) k.load(open(TEST_DEPLOYMENTS).read()) k._build_state() assert k.state_objects
def test_load_folder(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k._build_state() # check that no pods are orphan pods = list(filter(lambda x: isinstance(x, Pod), k.state_objects)) assert pods for pod in pods: assert pod.atNode._property_value != Node.NODE_NULL
def test_load_folder_load_pod_labels(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k._build_state() for ds in filter(lambda x: isinstance(x, Pod), k.state_objects): if labelFactory.get("app", "redis-evict") in ds.metadata_labels._get_value(): return raise Exception("Can not check labels load")
def test_spec_selector_labels(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k._build_state() for ds in filter(lambda x: isinstance(x, Service), k.state_objects): if labelFactory.get("app", "redis-evict") in ds.spec_selector._get_value(): return raise Exception("Can not check labels load")
def update(data=None): "Fetch information from currently selected cluster" if isinstance(data, io.IOBase): data = data.read() k = KubernetesCluster() if not data: global md5_cluster result = subprocess.Popen(cluster_md5_sh, shell=True, stdout=subprocess.PIPE, executable='/bin/bash') md5_cluster = result.stdout.read().decode('ascii').split()[0] assert len(md5_cluster ) == 32, "md5_cluster sum wrong len({0}) not is 32".format( md5_cluster) result = subprocess.run( ['kubectl', 'get', 'all', '--all-namespaces', '-o=json'], stdout=subprocess.PIPE) if len(result.stdout) < 100: print(result.stdout) raise SystemError( "Error using kubectl. Make sure `kubectl get pods` is working." ) data = json.loads(result.stdout.decode("utf-8")) for item in data["items"]: k.load_item(item) result = subprocess.run(['kubectl', 'get', 'node', '-o=json'], stdout=subprocess.PIPE) if len(result.stdout) < 100: raise SystemError( "Error using kubectl. Make sure `kubectl get pods` is working." ) data = json.loads(result.stdout.decode("utf-8")) for item in data["items"]: k.load_item(item) result = subprocess.run(['kubectl', 'get', 'pc', '-o=json'], stdout=subprocess.PIPE) if len(result.stdout) < 100: raise SystemError( "Error using kubectl. Make sure `kubectl get pods` is working." ) data = json.loads(result.stdout.decode("utf-8")) for item in data["items"]: k.load_item(item) else: for ys in kalc.misc.util.split_yamldumps(data): k.load(ys) k._build_state() global kalc_state_objects kalc_state_objects.clear() kalc_state_objects.extend(k.state_objects) global cluster cluster = next(filter(lambda x: isinstance(x, GlobalVar), k.state_objects))
def test_convert_node_problem(): # Initialize scheduler, globalvar k = KubernetesCluster() scheduler = next( filter(lambda x: isinstance(x, Scheduler), k.state_objects)) globalVar = next( filter(lambda x: isinstance(x, GlobalVar), k.state_objects)) # initial node state n = Node() n.cpuCapacity = 5 n.memCapacity = 5 # Create running pods pod_running_1 = build_running_pod(1, 2, 2, n) pod_running_2 = build_running_pod(2, 2, 2, n) ## Set consumptoin as expected n.currentFormalCpuConsumption = 4 n.currentFormalMemConsumption = 4 n.amountOfActivePods = 2 pc = PriorityClass() pc.priority = 10 pc.metadata_name = "high-prio-test" # Service to detecte eviction s = Service() s.metadata_name = "test-service" s.amountOfActivePods = 2 s.status = STATUS_SERV["Started"] # our service has multiple pods but we are detecting pods pending issue # remove service as we are detecting service outage by a bug above pod_running_1.targetService = s pod_running_2.targetService = s pod_running_1.hasService = True pod_running_2.hasService = True pod_running_1.priorityClass = pc pod_running_2.priorityClass = pc d = Deployment() d.spec_replicas = 2 d.amountOfActivePods = 2 pod_running_1.hasDeployment = True pod_running_2.hasDeployment = True d.podList.add(pod_running_1) d.podList.add(pod_running_2) k.state_objects.extend([n, pod_running_1, pod_running_2, s, d, pc]) k2 = KubernetesCluster() for y in convert_space_to_yaml(k.state_objects, wrap_items=True): # print(y) k2.load(y) k2._build_state() # TODO: test node outage exclusion
def test_exclude_regexp_unit(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DAEMONET).read()) k._build_state() mark_excluded(k.state_objects, "Service:front*", skip_check=False) for p in filter(lambda x: isinstance(x, Service), k.state_objects): if str(p.metadata_name) == "frontend": if p.searchable: raise ValueError("exclude doesn't work")
def test_service_load(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k._build_state() objects = filter(lambda x: isinstance(x, Service), k.state_objects) for p in objects: if p.metadata_name == "redis-master-create" and \ labelFactory.get("app", "redis-create") in p.metadata_labels._get_value(): return raise ValueError("Could not find service loded")
def test_load_requests(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DAEMONET).read()) k._build_state() objects = filter(lambda x: isinstance(x, DaemonSet), k.state_objects) for p in objects: if p.metadata_name == "fluentd-elasticsearch": return raise ValueError("Could not find service loded")
def test_load_priorityclass_custom_high_priority(): k = KubernetesCluster() k.load(open(TEST_PRIORITYCLASS).read()) k._build_state() found = False for pc in filter(lambda x: isinstance(x, PriorityClass), k.state_objects): if pc.metadata_name == "high-priority": # print(pc.priority, pc.priority._get_value()) assert pc.priority > 0 found = True assert found
def test_load_folder_create_labels(): k = KubernetesCluster() # k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DAEMONET).read()) k._build_state() for ds in filter(lambda x: isinstance(x, DaemonSet), k.state_objects): if labelFactory.get( "k8s-app", "fluentd-logging") in ds.metadata_labels._get_value(): return raise Exception("Can not check labels load")
def test_heapster_load(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k._build_state() heapsterpod = next(filter(lambda x: isinstance(x, Pod) and \ "heapster" in str(x.metadata_name), k.state_objects)) criticalpc = next(filter(lambda x: isinstance(x, PriorityClass) \ and x.metadata_name == "system-cluster-critical", k.state_objects)) # print(heapsterpod.priorityClass._get_value(), criticalpc._get_value()) assert heapsterpod.priorityClass._get_value() == criticalpc._get_value() assert heapsterpod.priorityClass == criticalpc
def test_pod_target_attached(): k = KubernetesCluster() k.load_dir(TEST_TARGET_DUMP) k.create_resource(open(TEST_TARGET_CREATE).read()) k._build_state() deployments = filter(lambda x: isinstance(x, Deployment), k.state_objects) for deployment in deployments: if deployment.metadata_name._get_value() == "redis-master-create": for pod in util.objDeduplicatorByName( deployment.podList._get_value()): assert pod.targetService._get_value() != None
def test_priority_is_loaded(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k._build_state() priorityClasses = filter(lambda x: isinstance(x, PriorityClass), k.state_objects) for p in priorityClasses: if p.metadata_name == "high-priority" and p.preemptionPolicy == POLICY["PreemptLowerPriority"]\ and p.priority > 0: return raise ValueError("Could not find priority loded")
def test_queue_status(): "test length and status of scheduler queue after load" k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DAEMONET).read()) k._build_state() scheduler = next( filter(lambda x: isinstance(x, Scheduler), k.state_objects)) nodes = list(filter(lambda x: isinstance(x, Node), k.state_objects)) # assert scheduler.queueLength == len(nodes) assert scheduler.podQueue._get_value() assert scheduler.status == STATUS_SCHED["Changed"]
def test_anyservice_interrupted_fromfiles(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DAEMONET).read()) k._build_state() mark_excluded_service(k.state_objects) p = AnyServiceInterrupted(k.state_objects) print_objects(k.state_objects) p.run(timeout=6600, sessionName="test_anyservice_interrupted_fromfiles") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml())
def test_load_pods_new(): k = KubernetesCluster() k.load(open(TEST_PRIORITYCLASS).read()) k.load(open(TEST_PODS).read()) k._build_state() # TODO: check if pod is fully loaded pod = k.state_objects[2 + 3] assert isinstance(pod, Pod) assert len(pod.metadata_labels._get_value()) > 0 assert pod.status == STATUS_POD["Running"] assert k.state_objects
def test_limits_for_pods_created(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DAEMONET).read()) k._build_state() objects = filter(lambda x: isinstance(x, Pod), k.state_objects) for p in objects: if str(p.metadata_name).startswith("fluentd-elasticsearch") and \ p.cpuRequest > -1 and \ p.memRequest > -1 and \ p.memLimit > -1: return raise ValueError("Could not find service loded")
def test_load_limits(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DEPLOYMENT).read()) k._build_state() objects = filter(lambda x: isinstance(x, Deployment), k.state_objects) for p in objects: if p.metadata_name == "redis-master" and \ p.cpuRequest > -1 and \ p.memRequest > -1 and \ p.memLimit > -1: return raise ValueError("Could not find service loded")
def test_load(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k._build_state() objects = filter(lambda x: isinstance(x, DaemonSet), k.state_objects) for p in objects: if p.metadata_name == "fluentd-elasticsearch": assert p.cpuRequest._get_value( ) == util.cpuConvertToAbstractProblem("400m") assert p.memRequest._get_value( ) == util.memConvertToAbstractProblem("400Mi") assert p.memLimit._get_value() == util.memConvertToAbstractProblem( "400Mi") return raise ValueError("Could not find service loded")
def test_anydeployment_interrupted_fromfiles(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DEPLOYMENT).read()) k._build_state() mark_excluded_service(k.state_objects) print("------Objects before solver processing------") print_objects(k.state_objects) p = NodeInterupted(k.state_objects) p.run(timeout=6600, sessionName="test_anydeployment_interrupted_fromfiles") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print("------Objects after solver processing------") print(Scenario(p.plan).asyaml()) print_objects(k.state_objects)
def test_load_load_create_exeption(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.create_resource(open(TEST_DEPLOYMENT).read()) try: k._build_state() except AssertionError as e: print(str(e)) assert str( e ) == "Error from server (AlreadyExists): deployments.apps \"redis-master\" already exists" objects = filter(lambda x: isinstance(x, Deployment), k.state_objects) for p in objects: if p.metadata_name == "redis-master": return raise ValueError("Could not find service loded")
def test_load_create(): k = KubernetesCluster() k.load_dir(TEST_CLUSTER_FOLDER) k.apply_resource(open(TEST_DAEMONSET_APPLY).read()) k._build_state() objects = filter(lambda x: isinstance(x, DaemonSet), k.state_objects) for p in objects: if p.metadata_name == "fluentd-elasticsearch": assert len(util.objDeduplicatorByName(p.podList._get_value())) == 2 assert p.cpuRequest._get_value( ) == util.cpuConvertToAbstractProblem("10m") assert p.memRequest._get_value( ) == util.memConvertToAbstractProblem("10Mi") assert p.memLimit._get_value() == util.memConvertToAbstractProblem( "10Mi") return raise ValueError("Could not find service loded")
def test_cyclic_load_1(): k, globalVar, n = prepare_test_single_node_dies_2pod_killed_service_outage() yamlState = convert_space_to_yaml(k.state_objects, wrap_items=True) k2 = KubernetesCluster() for y in yamlState: # print(y) k2.load(y) k2._build_state() globalVar = k2.state_objects[1] # print("--- RUN 2 ---") yamlState2 = convert_space_to_yaml(k2.state_objects, wrap_items=True) # for y in yamlState2: # print(y) ys1 = ''.join([i for i in repr(yamlState) if not i.isdigit()]) ys2 = ''.join([i for i in repr(yamlState2) if not i.isdigit()]) assert ys1 == ys2
def test_OptimisticRun(): k = KubernetesCluster() k.load(open(NODE1).read()) k.load(open(NODE2).read()) k.load(open(PODS).read()) # k.load(open(PODS_PENDING).read()) k.load(open(SERVICES).read()) k.load(open(REPLICASETS).read()) k.load(open(PRIORITYCLASSES).read()) k.load(open(DEPLOYMENT).read()) k.create_resource(open(DEPLOYMENT_NEW).read()) k._build_state() p = OptimisticRun( k.state_objects) # self.scheduler.status == STATUS_SCHED["Clean"] # print_objects(k.state_objects) p.run(timeout=6600, sessionName="test_OptimisticRun") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml())
def run_dir_wo_cli(DUMP_local,CHANGE_local): k = KubernetesCluster() if not (DUMP_local is None): for dump_item in DUMP_local: k.load_dir(dump_item) if not (CHANGE_local is None): for change_item in CHANGE_local: k.create_resource(open(change_item).read()) k._build_state() p = OptimisticRun(k.state_objects) print("#### run_wo_cli:") print("#### print_objects before run: #####") print(print_objects(k.state_objects)) p.run(timeout=999000, sessionName="test_OptimisticRun") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print("#### print_objects after run: ######") print(print_objects(k.state_objects))
def test_pod_cant_start(): k = KubernetesCluster() k.load(open(NODE1).read()) # trim resource, run only one Node k.load(open(PODS).read()) k.load(open(SERVICES).read()) k.load(open(REPLICASETS).read()) k.load(open(PRIORITYCLASSES).read()) k.load(open(DEPLOYMENT).read()) k.create_resource(open(DEPLOYMENT_NEW_WO_PRIO).read()) k._build_state() p = OptimisticRun( k.state_objects ) # TODO check me, i'd like to run exiction test with killpod execution # print_objects(k.state_objects) p.run(timeout=6600, sessionName="test_pod_cant_start") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml())
def test_cyclic_create(): k, globalVar, n = prepare_test_single_node_dies_2pod_killed_service_outage() yamlStateBeforeCreate = convert_space_to_yaml(k.state_objects, wrap_items=True) pod_pending_1 = build_pending_pod(3,2,2,n) dnew = Deployment() dnew.amountOfActivePods = 0 dnew.spec_replicas = 1 dnew.podList.add(pod_pending_1) # important to add as we extract status, priority spec from pod snew = Service() snew.metadata_name = "test-service-new" snew.amountOfActivePods = 0 pod_pending_1.targetService = snew create_objects = [dnew, snew] yamlCreate = convert_space_to_yaml(create_objects, wrap_items=False, load_logic_support=False) # snew.status = STATUS_SERV["Started"] k.state_objects.extend([pod_pending_1, dnew, snew]) yamlState = convert_space_to_yaml(k.state_objects, wrap_items=True) k2 = KubernetesCluster() for y in yamlStateBeforeCreate: # print(y) k2.load(y) for y in yamlCreate: k2.load(y, mode=KubernetesCluster.CREATE_MODE) k2._build_state() globalVar = k2.state_objects[1] # print("--- RUN 2 ---") yamlState2 = convert_space_to_yaml(k2.state_objects, wrap_items=True) # for y in yamlState2: # print(y) assert prepare_yamllist_for_diff(yamlState, ignore_names=True) == \ prepare_yamllist_for_diff(yamlState2, ignore_names=True)