def update(data=None): "Fetch information from currently selected cluster" if isinstance(data, io.IOBase): data = data.read() k = KubernetesCluster() if not data: global md5_cluster result = subprocess.Popen(cluster_md5_sh, shell=True, stdout=subprocess.PIPE, executable='/bin/bash') md5_cluster = result.stdout.read().decode('ascii').split()[0] assert len(md5_cluster ) == 32, "md5_cluster sum wrong len({0}) not is 32".format( md5_cluster) result = subprocess.run( ['kubectl', 'get', 'all', '--all-namespaces', '-o=json'], stdout=subprocess.PIPE) if len(result.stdout) < 100: print(result.stdout) raise SystemError( "Error using kubectl. Make sure `kubectl get pods` is working." ) data = json.loads(result.stdout.decode("utf-8")) for item in data["items"]: k.load_item(item) result = subprocess.run(['kubectl', 'get', 'node', '-o=json'], stdout=subprocess.PIPE) if len(result.stdout) < 100: raise SystemError( "Error using kubectl. Make sure `kubectl get pods` is working." ) data = json.loads(result.stdout.decode("utf-8")) for item in data["items"]: k.load_item(item) result = subprocess.run(['kubectl', 'get', 'pc', '-o=json'], stdout=subprocess.PIPE) if len(result.stdout) < 100: raise SystemError( "Error using kubectl. Make sure `kubectl get pods` is working." ) data = json.loads(result.stdout.decode("utf-8")) for item in data["items"]: k.load_item(item) else: for ys in kalc.misc.util.split_yamldumps(data): k.load(ys) k._build_state() global kalc_state_objects kalc_state_objects.clear() kalc_state_objects.extend(k.state_objects) global cluster cluster = next(filter(lambda x: isinstance(x, GlobalVar), k.state_objects))
def test_convert_node_problem(): # Initialize scheduler, globalvar k = KubernetesCluster() scheduler = next( filter(lambda x: isinstance(x, Scheduler), k.state_objects)) globalVar = next( filter(lambda x: isinstance(x, GlobalVar), k.state_objects)) # initial node state n = Node() n.cpuCapacity = 5 n.memCapacity = 5 # Create running pods pod_running_1 = build_running_pod(1, 2, 2, n) pod_running_2 = build_running_pod(2, 2, 2, n) ## Set consumptoin as expected n.currentFormalCpuConsumption = 4 n.currentFormalMemConsumption = 4 n.amountOfActivePods = 2 pc = PriorityClass() pc.priority = 10 pc.metadata_name = "high-prio-test" # Service to detecte eviction s = Service() s.metadata_name = "test-service" s.amountOfActivePods = 2 s.status = STATUS_SERV["Started"] # our service has multiple pods but we are detecting pods pending issue # remove service as we are detecting service outage by a bug above pod_running_1.targetService = s pod_running_2.targetService = s pod_running_1.hasService = True pod_running_2.hasService = True pod_running_1.priorityClass = pc pod_running_2.priorityClass = pc d = Deployment() d.spec_replicas = 2 d.amountOfActivePods = 2 pod_running_1.hasDeployment = True pod_running_2.hasDeployment = True d.podList.add(pod_running_1) d.podList.add(pod_running_2) k.state_objects.extend([n, pod_running_1, pod_running_2, s, d, pc]) k2 = KubernetesCluster() for y in convert_space_to_yaml(k.state_objects, wrap_items=True): # print(y) k2.load(y) k2._build_state() # TODO: test node outage exclusion
def test_load_priorityclass_custom_high_priority(): k = KubernetesCluster() k.load(open(TEST_PRIORITYCLASS).read()) k._build_state() found = False for pc in filter(lambda x: isinstance(x, PriorityClass), k.state_objects): if pc.metadata_name == "high-priority": # print(pc.priority, pc.priority._get_value()) assert pc.priority > 0 found = True assert found
def test_load_pods_new(): k = KubernetesCluster() k.load(open(TEST_PRIORITYCLASS).read()) k.load(open(TEST_PODS).read()) k._build_state() # TODO: check if pod is fully loaded pod = k.state_objects[2 + 3] assert isinstance(pod, Pod) assert len(pod.metadata_labels._get_value()) > 0 assert pod.status == STATUS_POD["Running"] assert k.state_objects
def test_load_deployments(): k = KubernetesCluster() k.load(open(TEST_PRIORITYCLASS).read()) k.load(open(TEST_NODES).read()) k.load(open(TEST_PODS).read()) k.load(open(TEST_DEPLOYMENTS).read()) k._build_state() assert k.state_objects
def test_cyclic_load_1(): k, globalVar, n = prepare_test_single_node_dies_2pod_killed_service_outage() yamlState = convert_space_to_yaml(k.state_objects, wrap_items=True) k2 = KubernetesCluster() for y in yamlState: # print(y) k2.load(y) k2._build_state() globalVar = k2.state_objects[1] # print("--- RUN 2 ---") yamlState2 = convert_space_to_yaml(k2.state_objects, wrap_items=True) # for y in yamlState2: # print(y) ys1 = ''.join([i for i in repr(yamlState) if not i.isdigit()]) ys2 = ''.join([i for i in repr(yamlState2) if not i.isdigit()]) assert ys1 == ys2
def run_wo_cli(DUMP_local,CHANGE_local): k = KubernetesCluster() if not (DUMP_local is None): for dump_item in DUMP_local: k.load(open(dump_item).read()) if not (CHANGE_local is None): for change_item in CHANGE_local: k.create_resource(open(change_item).read()) k._build_state() p = OptimisticRun(k.state_objects) print("#### run_wo_cli:") print("#### print_objects before run: ######") print(print_objects(k.state_objects)) p.run(timeout=999000, sessionName="test_OptimisticRun") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print("#### print_objects after run: ######") print(print_objects(k.state_objects))
def test_cyclic_create(): k, globalVar, n = prepare_test_single_node_dies_2pod_killed_service_outage() yamlStateBeforeCreate = convert_space_to_yaml(k.state_objects, wrap_items=True) pod_pending_1 = build_pending_pod(3,2,2,n) dnew = Deployment() dnew.amountOfActivePods = 0 dnew.spec_replicas = 1 dnew.podList.add(pod_pending_1) # important to add as we extract status, priority spec from pod snew = Service() snew.metadata_name = "test-service-new" snew.amountOfActivePods = 0 pod_pending_1.targetService = snew create_objects = [dnew, snew] yamlCreate = convert_space_to_yaml(create_objects, wrap_items=False, load_logic_support=False) # snew.status = STATUS_SERV["Started"] k.state_objects.extend([pod_pending_1, dnew, snew]) yamlState = convert_space_to_yaml(k.state_objects, wrap_items=True) k2 = KubernetesCluster() for y in yamlStateBeforeCreate: # print(y) k2.load(y) for y in yamlCreate: k2.load(y, mode=KubernetesCluster.CREATE_MODE) k2._build_state() globalVar = k2.state_objects[1] # print("--- RUN 2 ---") yamlState2 = convert_space_to_yaml(k2.state_objects, wrap_items=True) # for y in yamlState2: # print(y) assert prepare_yamllist_for_diff(yamlState, ignore_names=True) == \ prepare_yamllist_for_diff(yamlState2, ignore_names=True)
def test_single_node_dies_2pod_killed_service_outage_invload(): k, globalVar = prepare_test_single_node_dies_2pod_killed_service_outage() yamlState = convert_space_to_yaml(k.state_objects, wrap_items=True) k2 = KubernetesCluster() for y in yamlState: # print(y) k2.load(y) k2._build_state() globalVar = k2.state_objects[1] class Task_check_services(Check_services): goal = lambda self: globalVar.is_node_disrupted == True \ and globalVar.is_service_disrupted == True p = Task_check_services(k2.state_objects) p.run(timeout=200) print_plan(p) assert "NodeOutageFinished" in "\n".join([repr(x) for x in p.plan]) assert "Initiate_killing_of_Pod_because_of_node_outage" in "\n".join( [repr(x) for x in p.plan]) assert "MarkServiceOutageEvent" in "\n".join([repr(x) for x in p.plan])
def reload_cluster_from_yaml(k, create_objects): perform_yaml_test = True try: yamlState = convert_space_to_yaml(k.state_objects, wrap_items=True) except Exception as e: print("yaml conertion error",e) perform_yaml_test = False try: yamlCreate = convert_space_to_yaml(create_objects, wrap_items=False, load_logic_support=False) except Exception as e: print("yaml 2 conertion error",e) perform_yaml_test = False k2 = KubernetesCluster() if perform_yaml_test: for y in yamlState: k2.load(y) if perform_yaml_test: for y in yamlCreate: k2.load(y, mode=KubernetesCluster.CREATE_MODE) k2._build_state() return k2
def run_wo_cli_step1(DUMP_local,CHANGE_local): k = KubernetesCluster() if not (DUMP_local is None): for dump_item in DUMP_local: k.load(open(dump_item).read()) if not (CHANGE_local is None): for change_item in CHANGE_local: k.create_resource(open(change_item).read()) k._build_state() pod_running = next(filter(lambda x: isinstance(x, Pod) and x.status == STATUS_POD["Running"], k.state_objects)) class NewGoal(OptimisticRun): goal = lambda self: pod_running.status == STATUS_POD["Killing"] p = NewGoal(k.state_objects) print("#### run_wo_cli:") print("#### print_objects before run: #### ") print(print_objects(k.state_objects)) p.run(timeout=999000, sessionName="test_OptimisticRun") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print("#### print_objects after run: ####") print(print_objects(k.state_objects))
def test_OptimisticRun(): k = KubernetesCluster() k.load(open(NODE1).read()) k.load(open(NODE2).read()) k.load(open(PODS).read()) # k.load(open(PODS_PENDING).read()) k.load(open(SERVICES).read()) k.load(open(REPLICASETS).read()) k.load(open(PRIORITYCLASSES).read()) k.load(open(DEPLOYMENT).read()) k.create_resource(open(DEPLOYMENT_NEW).read()) k._build_state() p = OptimisticRun( k.state_objects) # self.scheduler.status == STATUS_SCHED["Clean"] # print_objects(k.state_objects) p.run(timeout=6600, sessionName="test_OptimisticRun") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml())
def test_pod_cant_start(): k = KubernetesCluster() k.load(open(NODE1).read()) # trim resource, run only one Node k.load(open(PODS).read()) k.load(open(SERVICES).read()) k.load(open(REPLICASETS).read()) k.load(open(PRIORITYCLASSES).read()) k.load(open(DEPLOYMENT).read()) k.create_resource(open(DEPLOYMENT_NEW_WO_PRIO).read()) k._build_state() p = OptimisticRun( k.state_objects ) # TODO check me, i'd like to run exiction test with killpod execution # print_objects(k.state_objects) p.run(timeout=6600, sessionName="test_pod_cant_start") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml())
def test_start_pod(): k = KubernetesCluster() k.load(open(NODE1).read()) k.load(open(NODE2).read()) k.load(open(PODS).read()) # k.load(open(PODS_PENDING).read()) k.load(open(SERVICES).read()) k.load(open(REPLICASETS).read()) k.load(open(PRIORITYCLASSES).read()) k.load(open(DEPLOYMENT).read()) k.create_resource(open(DEPLOYMENT_NEW).read()) k._build_state() class PodStart(K8ServiceInterruptSearch): goal = lambda self: self.goalFoo() def goalFoo(self): for pod in filter(lambda x: isinstance(x, mpod.Pod), k.state_objects): if pod.status != STATUS_POD["Running"]: return False return True p = PodStart( k.state_objects) # self.scheduler.status == STATUS_SCHED["Clean"] # print_objects(k.state_objects) p.run(timeout=6600, sessionName="test_start_pods") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml()) assert "StartPod" in p.plan.__str__( ) # test for transition from Pending to Running pods = filter(lambda x: isinstance(x, mpod.Pod), k.state_objects) nodes = filter(lambda x: isinstance(x, Node), k.state_objects) for pod in pods: assert pod.atNode in nodes._get_value()
def test_start_pod_from_dump(): k = KubernetesCluster() k.load(open(NODE1).read()) k.load(open(NODE2).read()) k.load(open(PODS_PENDING).read()) k.load(open(SERVICES).read()) k.load(open(REPLICASETS).read()) k.load(open(PRIORITYCLASSES).read()) k.load(open(DEPLOYMENT).read()) k._build_state() p = QueueLoadCheck( k.state_objects) # self.scheduler.status == STATUS_SCHED["Clean"] # print_objects(k.state_objects) p.run(timeout=6600, sessionName="test_start_pod_from_dump") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml()) assert "StartPod" in p.plan.__str__( ) # test for transition from Pending to Running pods = filter(lambda x: isinstance(x, mpod.Pod), k.state_objects) nodes = filter(lambda x: isinstance(x, Node), k.state_objects) for pod in pods: assert pod.atNode in nodes._get_value( ) # check each pod than each have atNode
def test_load_nodes(): k = KubernetesCluster() k.load(open(TEST_NODES).read()) k._build_state() assert k.state_objects
def test_load_pods(): k = KubernetesCluster() k.load(open(TEST_PRIORITYCLASS).read()) k.load(open(TEST_PODS).read()) k._build_state() assert k.state_objects
def test_load_deployment(): k = KubernetesCluster() k.load( open("./tests/client-cases/criticalhopmanifest_prefixed.yaml").read()) k._build_state() print_objects(k.state_objects)
def test_killpod(): k = KubernetesCluster() k.load(open(NODE1).read()) # trim resource, run only one Node k.load(open(PODS).read()) k.load(open(SERVICES).read()) k.load(open(REPLICASETS).read()) k.load(open(PRIORITYCLASSES).read()) k.load(open(DEPLOYMENT).read()) k.create_resource(open(DEPLOYMENT_NEW).read()) k._build_state() p = OptimisticRun( k.state_objects ) # TODO check me, i'd like to run exiction test with killpod execution # print_objects(k.state_objects) p.run(timeout=6600, sessionName="test_start_pods") if not p.plan: raise Exception("Could not solve %s" % p.__class__.__name__) print(Scenario(p.plan).asyaml()) assert "StartPod" in p.plan.__str__( ) # test for transition from Pending to Running #get pods only in Running state to check atNode value runningPods = filter( lambda z: z.status != STATUS_POD["Running"], (filter(lambda x: isinstance(x, mpod.Pod), k.state_objects))) nodes = filter(lambda x: isinstance(x, Node), k.state_objects) for pod in runningPods: assert pod.atNode in nodes._get_value( ) # check each pod than each have atNode killingPods = filter( lambda z: z.status != STATUS_POD["Killing"], (filter(lambda x: isinstance(x, mpod.Pod), k.state_objects))) assert len(killingPods) > 0 # test that some pod Killed