def test__host_maintenance_lifecycle(host_affinity_job, maintenance): # Pick a host that is UP and start maintenance on it test_host = get_host_in_state(hpb.HOST_STATE_UP) # Set host affinity of the job to the selected host host_affinity_job.job_config.defaultConfig.\ constraint.labelConstraint.label.value = test_host host_affinity_job.create() # Start maintenance on the selected host resp = maintenance['start']([test_host]) assert resp assert is_host_in_state(test_host, hpb.HOST_STATE_DRAINING) # Wait for host to transition to DOWN wait_for_host_state(test_host, hpb.HOST_STATE_DOWN) # Complete maintenance on the test hosts resp = maintenance['stop']([test_host]) assert resp # Host should no longer be DOWN assert not is_host_in_state(test_host, hpb.HOST_STATE_DOWN) wait_for_host_state(test_host, hpb.HOST_STATE_UP)
def test__host_maintenance_violate_sla(stateless_job, maintenance): """ 1. Create a stateless job(instance_count=4) with host-limit-1 constraint and MaximumUnavailableInstances=1. This means that there one instance that is unavailable. 2. Start host maintenance on one of the hosts (say A). 3. Since one instance is already unavailable, no more instances should be killed due to host maintenance. Verify that host A does not transition to DOWN. """ job_spec_dump = load_test_config('test_stateless_job_spec_sla.yaml') json_format.ParseDict(job_spec_dump, stateless_job.job_spec) stateless_job.job_spec.instance_count = 4 stateless_job.create() stateless_job.wait_for_all_pods_running(num_pods=3) # Pick a host that is UP and start maintenance on it test_host1 = get_host_in_state(host_pb2.HOST_STATE_UP) resp = maintenance["start"]([test_host1]) assert resp try: wait_for_host_state(test_host1, host_pb2.HOST_STATE_DOWN) assert False, 'Host should not transition to DOWN' except: assert is_host_in_state(test_host1, host_pb2.HOST_STATE_DRAINING) assert len( stateless_job.query_pods(states=[pod_pb2.POD_STATE_RUNNING])) == 3
def test__host_maintenance_within_sla_limit(stateless_job, maintenance): """ 1. Create a stateless job(instance_count=4) and MaximumUnavailableInstances=1. Wait for all pods to reach RUNNING state. This means there is at least one host with more than one instance. 2. Start host maintenance on a host (say A) with more than 1 instance. 3. Pods on the host A should get killed in a way (1 at a time) that doesn't violate the SLA and host A should transition to DOWN """ stateless_job.job_spec.instance_count = 4 stateless_job.job_spec.sla.maximum_unavailable_instances = 1 stateless_job.create() stateless_job.wait_for_all_pods_running() hosts = [h.hostname for h in query_hosts([]).host_infos] host_to_task_count = get_host_to_task_count(hosts, stateless_job) sorted_hosts = [ t[0] for t in sorted(host_to_task_count.items(), key=operator.itemgetter(1)) ] # Pick a host that has pods running on it and start maintenance on it. test_host = sorted_hosts[0] resp = maintenance["start"]([test_host]) assert resp # Wait for host to transition to DOWN attempts = 0 max_retry_attempts = 20 log.info( "%s waiting for state %s", test_host, host_pb2.HostState.Name(host_pb2.HOST_STATE_DOWN), ) while attempts < max_retry_attempts: try: if is_host_in_state(test_host, host_pb2.HOST_STATE_DOWN): break # if the number of available pods is less than 2 (instance_count - # maximum_unavailable_instances) fail the test if len(stateless_job.query_pods( states=[pod_pb2.POD_STATE_RUNNING])) < 2: assert False except Exception as e: log.warn(e) finally: time.sleep(5) attempts += 1 if attempts == max_retry_attempts: log.info( "%s max attempts reached to wait for host state %s", test_host, host_pb2.HostState.Name(host_pb2.HOST_STATE_DOWN), ) assert False
def test__update_with_host_maintenance_and_agent_down(stateless_job, maintenance): """ 1. Create a large stateless job (that take up more than two-thirds of the cluster resources) with MaximumUnavailableInstances=2. 2. Start host maintenance on one of the hosts (say A) having pods of the job. MaximumUnavailableInstances=2 ensures that not more than 2 pods are unavailable due to host maintenance at a time. 3. Take down another host which has pods running on it. This will TASK_LOST to be sent for all pods on the host after 75 seconds. 4. Start an update to modify the instance spec of one of the pods. 5. Since TASK_LOST would cause the job SLA to be violated, instances on the host A should not be killed once LOST event is received. Verify that host A does not transition to DOWN. """ stateless_job.job_spec.instance_count = 30 stateless_job.job_spec.default_spec.containers[0].resource.cpu_limit = 0.3 stateless_job.job_spec.sla.maximum_unavailable_instances = 2 stateless_job.create() stateless_job.wait_for_all_pods_running() hosts = [h.hostname for h in query_hosts([]).host_infos] host_to_task_count = get_host_to_task_count(hosts, stateless_job) sorted_hosts = [ t[0] for t in sorted(host_to_task_count.items(), key=operator.itemgetter(1), reverse=True) ] # Pick a host that has pods running on it to start maintenance on it. test_host = sorted_hosts[0] # pick another host which has pods of the job to take down host_container = get_container([sorted_hosts[1]]) try: host_container.stop() maintenance["start"]([test_host]) stateless_job.job_spec.instance_spec[10].containers.extend([ pod_pb2.ContainerSpec(resource=pod_pb2.ResourceSpec( disk_limit_mb=20)) ]) update = StatelessUpdate(stateless_job, updated_job_spec=stateless_job.job_spec, batch_size=0) update.create() update.wait_for_state(goal_state="SUCCEEDED") stateless_job.stop() wait_for_host_state(test_host, host_pb2.HOST_STATE_DOWN) assert False, 'Host should not transition to DOWN' except: assert is_host_in_state(test_host, host_pb2.HOST_STATE_DRAINING) pass finally: host_container.start()
def test__update_with_host_maintenance__bad_config(stateless_job, maintenance): """ 1. Create a stateless job with 6 instances. Wait for all instances to reach RUNNING state. This means that there is at least one host with 2 or more instances on it 2. Start a bad job update with max failure tolerance of 1 and auto-rollback disabled. 3. Start host maintenance on one of the hosts (say host A). 4. Wait for the update to fail. There should be 2 instances unavailable. 5. Since 2 instances are already unavailable and maximum_unavailable_instances=1, host maintenance should not proceed. Verify that the host A doesn't transition to DOWN. """ stateless_job.job_spec.sla.maximum_unavailable_instances = 1 stateless_job.job_spec.instance_count = 6 stateless_job.create() stateless_job.wait_for_all_pods_running() hosts = [h.hostname for h in query_hosts([]).host_infos] host_to_task_count = get_host_to_task_count(hosts, stateless_job) sorted_hosts = [ t[0] for t in sorted(host_to_task_count.items(), key=operator.itemgetter(1), reverse=True) ] job_spec_dump = load_test_config(UPDATE_STATELESS_JOB_BAD_SPEC) updated_job_spec = JobSpec() json_format.ParseDict(job_spec_dump, updated_job_spec) updated_job_spec.instance_count = 6 updated_job_spec.sla.maximum_unavailable_instances = 1 update = StatelessUpdate( stateless_job, updated_job_spec=updated_job_spec, max_failure_instances=1, max_instance_attempts=1, batch_size=2, ) update.create() # Pick a host that has pods running on it to start maintenance on it. test_host = sorted_hosts[0] maintenance["start"]([test_host]) update.wait_for_state(goal_state="FAILED", failed_state="SUCCEEDED") try: wait_for_host_state(test_host, host_pb2.HOST_STATE_DOWN) assert False, 'Host should not transition to DOWN' except: assert is_host_in_state(test_host, host_pb2.HOST_STATE_DRAINING)
def test__host_maintenance_violate_sla_restart_jobmgr(self, failure_tester, maintenance): """ 1. Create a stateless job(instance_count=4) with host-limit-1 constraint and MaximumUnavailableInstances=1. Since there are only 3 UP hosts, one of the instances will not get placed (hence unavailable). 2. Start host maintenance on one of the hosts (say A). 3. Restart job manager. 4. Since one instance is already unavailable, no more instances should be killed due to host maintenance. Verify that host A does not transition to DOWN """ stateless_job = failure_tester.stateless_job() job_spec_dump = load_test_config('test_stateless_job_spec_sla.yaml') json_format.ParseDict(job_spec_dump, stateless_job.job_spec) stateless_job.job_spec.instance_count = 4 stateless_job.create() stateless_job.wait_for_all_pods_running(num_pods=3) # Pick a host that is UP and start maintenance on it test_host1 = get_host_in_state( host_pb2.HOST_STATE_UP, failure_tester.client) # update the client in maintenance fixture maintenance["update_client"](failure_tester.client) resp = maintenance["start"]([test_host1]) assert resp leader = failure_tester.fw.get_leader_info(failure_tester.jobmgr) assert leader assert 0 != failure_tester.fw.restart(failure_tester.jobmgr, "leader") failure_tester.wait_for_leader_change(failure_tester.jobmgr, leader) failure_tester.reset_client() stateless_job.client = failure_tester.client # update the client of maintainance maintenance["update_client"](failure_tester.client) try: wait_for_host_state(test_host1, host_pb2.HOST_STATE_DOWN) assert False, 'Host should not transition to DOWN' except: assert is_host_in_state(test_host1, host_pb2.HOST_STATE_DRAINING) assert len(stateless_job.query_pods( states=[pod_pb2.POD_STATE_RUNNING])) == 3