def prepare_test_single_node_dies_2pod_killed_service_outage():
# Initialize scheduler, globalvar
k = KubernetesCluster()
scheduler = next(filter(lambda x: isinstance(x, Scheduler), k.state_objects))
globalVar = next(filter(lambda x: isinstance(x, GlobalVar), k.state_objects))
# initial node state
n = Node()
n.cpuCapacity = 5
n.memCapacity = 5
# Create running pods
pod_running_1 = build_running_pod(1,2,2,n)
pod_running_2 = build_running_pod(2,2,2,n)
## Set consumptoin as expected
n.currentFormalCpuConsumption = 4
n.currentFormalMemConsumption = 4
n.amountOfActivePods = 2
# Service to detecte eviction
s = Service()
s.metadata_name = "test-service"
s.amountOfActivePods = 2
s.status = STATUS_SERV["Started"]
# our service has multiple pods but we are detecting pods pending issue
# remove service as we are detecting service outage by a bug above
pod_running_1.targetService = s
pod_running_2.targetService = s
pod_running_1.hasService = True
pod_running_2.hasService = True
k.state_objects.extend([n, pod_running_1, pod_running_2, s])
# print_objects(k.state_objects)
return k, globalVar, n
def test_convert_node_problem():
# Initialize scheduler, globalvar
k = KubernetesCluster()
scheduler = next(
filter(lambda x: isinstance(x, Scheduler), k.state_objects))
globalVar = next(
filter(lambda x: isinstance(x, GlobalVar), k.state_objects))
# initial node state
n = Node()
n.cpuCapacity = 5
n.memCapacity = 5
# Create running pods
pod_running_1 = build_running_pod(1, 2, 2, n)
pod_running_2 = build_running_pod(2, 2, 2, n)
## Set consumptoin as expected
n.currentFormalCpuConsumption = 4
n.currentFormalMemConsumption = 4
n.amountOfActivePods = 2
pc = PriorityClass()
pc.priority = 10
pc.metadata_name = "high-prio-test"
# Service to detecte eviction
s = Service()
s.metadata_name = "test-service"
s.amountOfActivePods = 2
s.status = STATUS_SERV["Started"]
# our service has multiple pods but we are detecting pods pending issue
# remove service as we are detecting service outage by a bug above
pod_running_1.targetService = s
pod_running_2.targetService = s
pod_running_1.hasService = True
pod_running_2.hasService = True
pod_running_1.priorityClass = pc
pod_running_2.priorityClass = pc
d = Deployment()
d.spec_replicas = 2
d.amountOfActivePods = 2
pod_running_1.hasDeployment = True
pod_running_2.hasDeployment = True
d.podList.add(pod_running_1)
d.podList.add(pod_running_2)
k.state_objects.extend([n, pod_running_1, pod_running_2, s, d, pc])
k2 = KubernetesCluster()
for y in convert_space_to_yaml(k.state_objects, wrap_items=True):
# print(y)
k2.load(y)
k2._build_state()
# TODO: test node outage exclusion
def prepare_many_pods_without_yaml(nodes_amount,node_capacity,pod2_amount,pod0_amount,pod2_2_amount,pod3_amount):
# Initialize scheduler, globalvar
k = KubernetesCluster()
scheduler = next(filter(lambda x: isinstance(x, Scheduler), k.state_objects))
# initial node state
i = 0
j = 0
nodes = []
pods = []
# Service to detecte eviction
s = Service()
s.metadata_name = "test-service"
s.amountOfActivePods = 0
s2 = Service()
s2.metadata_name = "test-service2"
s2.amountOfActivePods = 0
# create Deploymnent that we're going to detect failure of...
d = Deployment()
d.spec_replicas = 2
pod_id = 0
for i in range(nodes_amount):
node_item = Node("node"+str(i))
node_item.metadata_name = "node"+str(i)
node_item.cpuCapacity = node_capacity
node_item.memCapacity = node_capacity
node_item.isNull = False
node_item.status = STATUS_NODE["Active"]
nodes.append(node_item)
for j in range(pod2_amount):
pod_running_2 = build_running_pod_with_d(pod_id,2,2,node_item,None,None)
pod_id += 1
pod_running_2.hasService = True
pods.append(pod_running_2)
node_item.amountOfActivePods += 1
s.podList.add(pod_running_2)
s.amountOfActivePods += 1
s.status = STATUS_SERV["Started"]
for j in range(pod0_amount):
pod_running_0 = build_running_pod_with_d(pod_id,0,0,node_item,None,None)
pod_id += 1
pods.append(pod_running_0)
node_item.amountOfActivePods += 1
for j in range(pod2_2_amount):
pod_running_2 = build_running_pod_with_d(pod_id,2,2,node_item,None,None)
pod_id += 1
pods.append(pod_running_2)
node_item.amountOfActivePods += 1
for j in range(pod3_amount):
pod_running_2 = build_running_pod_with_d(pod_id,2,2,nodes[0],None,None)
pod_id += 1
pod_running_2.hasService = True
pods.append(pod_running_2)
nodes[0].amountOfActivePods += 1
s2.podList.add(pod_running_2)
s2.amountOfActivePods +=1
# priority for pod-to-evict
pc = PriorityClass()
pc.priority = 10
pc.metadata_name = "high-prio-test"
k.state_objects.extend(nodes)
k.state_objects.extend(pods)
k.state_objects.extend([pc, s, s2 ])
create_objects = []
k._build_state()
globalVar = next(filter(lambda x: isinstance(x, GlobalVar), k.state_objects))
scheduler = next(filter(lambda x: isinstance(x, Scheduler), k.state_objects))
class HypothesisysNode_k1(HypothesisysNode):
pass
p = HypothesisysNode_k1(k.state_objects)
HypothesisysNode_k1.__name__ = inspect.stack()[0].function
assert_conditions = ["MarkServiceOutageEvent",\
"Mark_node_outage_event"]
not_assert_conditions = []
print_objects(k.state_objects)
test_case = StateSet()
test_case.scheduler = scheduler
test_case.globalVar = globalVar
test_case.pods = pods
test_case.nodes = nodes
services = [s,s2]
test_case.services = services
return k, p, test_case
def test_node_killer_pod_with_service():
# value start stop step
node_amount_range = range(2, 5, 2)
pod_amount_range = range(16, 61, 1)
per_node_capacity_range = range(20, 41, 10)
search = True
assert_brake = False
csvfile = open("{0}_{1}.csv".format(inspect.stack()[1].function, sha[:7]), 'w')
csvwriter = csv.writer(csvfile, delimiter=';')
for node_capacity in per_node_capacity_range:
for node_amount in node_amount_range:
for pod_amount in pod_amount_range:
if pod_amount > (node_amount * node_capacity) : continue
# Initialize scheduler, globalvar
start = time.time()
k = KubernetesCluster()
scheduler = next(filter(lambda x: isinstance(x, Scheduler), k.state_objects))
# initial node state
i = 0
j = 0
nodes = []
pods_running = []
high = PriorityClass()
high.priority = 10
high.metadata_name = "high"
# low = PriorityClass()
# low.priority = 0
# low.metadata_name = "low"
s = Service()
s.metadata_name = "test-service"
s.amountOfActivePods = 0
s.status = STATUS_SERV["Started"]
s.isSearched = True
isSearched = True
pod_id=0
for i in range(node_amount):
node_item = Node("node"+str(i))
node_item.cpuCapacity = node_capacity
node_item.memCapacity = node_capacity
node_item.isNull = False
node_item.status = STATUS_NODE["Active"]
node_item.isSearched = isSearched
isSearched = False
nodes.append(node_item)
node_counter = 0
for j in range(pod_amount):
node_item = nodes[node_counter]
if node_item.currentFormalCpuConsumption == node_capacity:
break
pod_running = Pod()
pod_running.metadata_name = "pod_prio_0_{0}_{1}".format(i,j)
pod_running.cpuRequest = 1
pod_running.memRequest = 1
pod_running.atNode = node_item
pod_running.status = STATUS_POD["Running"]
pod_running.hasDeployment = False
pod_running.hasService = False
pod_running.hasDaemonset = False
pod_running.priorityClass = high
pod_running.hasService = True
pods_running.append(pod_running)
# node_item.podList.add(pod_running)
node_item.currentFormalCpuConsumption += 1
node_item.currentFormalMemConsumption += 1
node_item.amountOfActivePods += 1
s.podList.add(pod_running)
s.amountOfActivePods += 1
node_counter += 1
if node_counter == len(nodes):
node_counter=0
k.state_objects.extend(nodes)
k.state_objects.extend(pods_running)
# k.state_objects.extend([low])
k.state_objects.append(high)
k.state_objects.append(s)
k._build_state()
print("(node_capacity * (node_amount - 1))(",(node_capacity * (node_amount - 1)), ")<(", pod_amount,")pod_amount")
if (node_capacity * (node_amount - 1)) < pod_amount:
task_type = "no-outage"
else:
task_type = "NodeOutageFinished"
print("check break node_amount {0} with capacity {1} pod amount {2}".format( node_amount, node_capacity,pod_amount))
print("-------------------")
print_objects(k.state_objects)
GenClass = type("{0}_{1}_{2}_{3}".format(inspect.stack()[1].function, node_amount, pod_amount, sha[:7]),(HypothesisysNode,),{})
p = GenClass(k.state_objects)
try:
p.run(timeout=1000, sessionName=f"gen_test_{node_capacity}_{node_amount}_{pod_amount}_L{LIN_COUNT}")
except Exception as e:
print("run break exception is \n",e)
assert False
# print_plan(p)
end = time.time()
print("-------------------")
print("timer :", int(end - start))
if p.plan != None:
csvwriter.writerow([node_amount, node_capacity, pod_amount, int(end - start), "ok"])
else:
csvwriter.writerow([node_amount, node_capacity, pod_amount, int(end - start), "empty_plan"])
csvfile.flush()
print("-------------------")
def test_1_1pod_2nodes_Service_outage():
# Initialize scheduler, globalvar
k = KubernetesCluster()
globalVar = next(
filter(lambda x: isinstance(x, GlobalVar), k.state_objects))
scheduler = next(
filter(lambda x: isinstance(x, Scheduler), k.state_objects))
# initial node state
i = 0
j = 0
nodes = []
pods = []
# Service to detecte eviction
s = Service()
s.metadata_name = "test-service"
s.amountOfActivePods = 0
s2 = Service()
s2.metadata_name = "test-service2"
s2.amountOfActivePods = 0
# create Deploymnent that we're going to detect failure of...
pod_id = 1
node_1 = Node("node 1")
node_1.cpuCapacity = 4
node_1.memCapacity = 4
node_1.isNull = False
node_1.status = STATUS_NODE["Active"]
node_2 = Node("node 2")
node_2.cpuCapacity = 4
node_2.memCapacity = 4
node_2.isNull = False
node_2.status = STATUS_NODE["Active"]
pod_running_1 = build_running_pod_with_d(pod_id, 2, 2, node_1, None, None)
pod_running_1.hasService = True
node_1.amountOfActivePods += 1
s.podList.add(pod_running_1)
s.amountOfActivePods += 1
s.status = STATUS_SERV["Started"]
# k.state_objects += [node_1,node_2,pod_running_1, s]
k.state_objects += [
node_1, node_2, pod_running_1, s, STATUS_POD["Pending"],
STATUS_POD["Killing"], STATUS_POD["Running"]
]
create_objects = []
k._build_state()
class HypothesisysNode_k1(HypothesisysNode):
pass
p = HypothesisysNode_k1(k.state_objects)
HypothesisysNode_k1.__name__ = inspect.stack()[0].function
not_assert_conditions = []
print_objects(k.state_objects)
p.Initiate_node_outage(node_1, globalVar)
# p.Initiate_killing_of_Pod_because_of_node_outage(node_1,pod_running_1,globalVar)
# p.KillPod_IF_Deployment_isNUll_Service_isNotNull_Daemonset_isNull(pod_running_1,node_1,s,scheduler)
# p.NodeOutageFinished(node_1,globalVar)
# p.Mark_node_outage_event(node_1,globalVar)
# p.SelectNode(pod_running_1,node_2,globalVar)
# p.StartPod_IF_Deployment_isNUll_Service_isNotNull_Daemonset_isNull(pod_running_1,node_2,scheduler,s,globalVar)
# p.SchedulerCleaneduler,globalVar)
print(" >> changed state << ")
print_objects(k.state_objects)
p.run()
print(" >> after << ")
print_objects(k.state_objects)
print_plan(p)