def test__host_maintenance_violate_sla(stateless_job, maintenance): """ 1. Create a stateless job(instance_count=4) with host-limit-1 constraint and MaximumUnavailableInstances=1. This means that there one instance that is unavailable. 2. Start host maintenance on one of the hosts (say A). 3. Since one instance is already unavailable, no more instances should be killed due to host maintenance. Verify that host A does not transition to DOWN. """ job_spec_dump = load_test_config('test_stateless_job_spec_sla.yaml') json_format.ParseDict(job_spec_dump, stateless_job.job_spec) stateless_job.job_spec.instance_count = 4 stateless_job.create() stateless_job.wait_for_all_pods_running(num_pods=3) # Pick a host that is UP and start maintenance on it test_host1 = get_host_in_state(host_pb2.HOST_STATE_UP) resp = maintenance["start"]([test_host1]) assert resp try: wait_for_host_state(test_host1, host_pb2.HOST_STATE_DOWN) assert False, 'Host should not transition to DOWN' except: assert is_host_in_state(test_host1, host_pb2.HOST_STATE_DRAINING) assert len( stateless_job.query_pods(states=[pod_pb2.POD_STATE_RUNNING])) == 3
def test__in_place_update_host_maintenance(stateless_job, maintenance): # add enough instances so each host should have some tasks running stateless_job.job_spec.instance_count = 9 # need extra retry attempts, since in-place update would need more time # to process given agent is put in maintenance mode stateless_job.config = IntegrationTestConfig( max_retry_attempts=300, pool_file='test_stateless_respool.yaml', ), stateless_job.create() stateless_job.wait_for_all_pods_running() job_spec_dump = load_test_config(UPDATE_STATELESS_JOB_SPEC) updated_job_spec = JobSpec() json_format.ParseDict(job_spec_dump, updated_job_spec) updated_job_spec.instance_count = 9 update = StatelessUpdate(stateless_job, updated_job_spec=updated_job_spec, batch_size=0) update.create(in_place=True) # Pick a host that is UP and start maintenance on it test_host = get_host_in_state(host_pb2.HOST_STATE_UP) resp = maintenance["start"]([test_host]) assert resp wait_for_host_state(test_host, host_pb2.HOST_STATE_DOWN) update.wait_for_state(goal_state="SUCCEEDED")
def test__update_with_sla_aware_host_maintenance(stateless_job, maintenance): """ 1. Create a stateless job with 3 instances. 2. Create a job update to update the instance job with instance count 2, add host-limit-1 constraint and define sla with maximum_unavailable_instances=1 3. Start host maintenance on one of the hosts 4. The host should transition to DOWN and the update workflow should SUCCEED """ stateless_job.create() stateless_job.wait_for_all_pods_running() job_spec_dump = load_test_config('test_stateless_job_spec_sla.yaml') updated_job_spec = JobSpec() json_format.ParseDict(job_spec_dump, updated_job_spec) updated_job_spec.instance_count = 2 update = StatelessUpdate(stateless_job, updated_job_spec=updated_job_spec, batch_size=1) update.create() # Pick a host that is UP and start maintenance on it test_host = get_host_in_state(host_pb2.HOST_STATE_UP) resp = maintenance["start"]([test_host]) assert resp update.wait_for_state(goal_state="SUCCEEDED") wait_for_host_state(test_host, host_pb2.HOST_STATE_DOWN)
def test__host_maintenance_lifecycle(host_affinity_job, maintenance): # Pick a host that is UP and start maintenance on it test_host = get_host_in_state(hpb.HOST_STATE_UP) # Set host affinity of the job to the selected host host_affinity_job.job_config.defaultConfig.constraint.labelConstraint.label.value = ( test_host) host_affinity_job.create() # Start maintenance on the selected host resp = maintenance["start"]([test_host]) assert resp assert is_host_in_state(test_host, hpb.HOST_STATE_DRAINING) # Wait for host to transition to DOWN wait_for_host_state(test_host, hpb.HOST_STATE_DOWN) # Complete maintenance on the test hosts resp = maintenance["stop"]([test_host]) assert resp # Host should no longer be DOWN assert not is_host_in_state(test_host, hpb.HOST_STATE_DOWN) wait_for_host_state(test_host, hpb.HOST_STATE_UP)
def test__start_maintenance_kill_tasks(host_affinity_job, maintenance): # Pick a host that is UP and start maintenance on it test_host = get_host_in_state(hpb.HOST_STATE_UP) # Set host affinity of the job to the selected host host_affinity_job.job_config.defaultConfig.constraint.labelConstraint.label.value = ( test_host) host_affinity_job.create() host_affinity_job.wait_for_state(goal_state="RUNNING") def all_running(): return all(t.state == task.RUNNING for t in host_affinity_job.get_tasks().values()) host_affinity_job.wait_for_condition(all_running) constraint = host_affinity_job.job_config.defaultConfig.constraint test_host = constraint.labelConstraint.label.value resp = maintenance["start"]([test_host]) assert resp def all_pending(): return all(t.state == task.PENDING for t in host_affinity_job.get_tasks().values()) # Wait for tasks to be killed and restarted host_affinity_job.wait_for_condition(all_pending)
def test__host_draining_resumes_on_hostmgr_recovery(host_affinity_job, maintenance, resmgr, hostmgr): # Pick a host that is UP and start maintenance on it test_host = get_host_in_state(hpb.HOST_STATE_UP) # Set host affinity of the job to the selected host host_affinity_job.job_config.defaultConfig.constraint.labelConstraint.label.value = ( test_host) host_affinity_job.create() host_affinity_job.wait_for_state(goal_state="RUNNING") def all_running(): return all(t.state == task.RUNNING for t in host_affinity_job.get_tasks().values()) host_affinity_job.wait_for_condition(all_running) constraint = host_affinity_job.job_config.defaultConfig.constraint test_host = constraint.labelConstraint.label.value # Stop resmgr to ensure maintenance queue is not polled resmgr.stop() resp = maintenance["start"]([test_host]) assert resp hostmgr.restart() resmgr.start() # Wait for host to transition to DOWN wait_for_host_state(test_host, hpb.HOST_STATE_DOWN)
def test__host_draining_resumes_on_resmgr_recovery( host_affinity_job, maintenance, jobmgr, resmgr, ): # Pick a host that is UP and start maintenance on it test_host = get_host_in_state(hpb.HOST_STATE_UP) # Set host affinity of the job to the selected host host_affinity_job.job_config.defaultConfig.constraint.labelConstraint.label.value = ( test_host) host_affinity_job.create() def all_running(): return all(t.state == task.RUNNING for t in host_affinity_job.get_tasks().values()) host_affinity_job.wait_for_condition(all_running) constraint = host_affinity_job.job_config.defaultConfig.constraint test_host = constraint.labelConstraint.label.value resp = maintenance["start"]([test_host]) assert resp # Stop jobmgr to ensure tasks are not killed jobmgr.stop() # Sleep for one draining period to ensure maintenance queue is polled time.sleep(draining_period_sec) resmgr.restart() jobmgr.start() # Wait for host to transition to DOWN wait_for_host_state(test_host, hpb.HOST_STATE_DOWN)
def test__host_maintenance_violate_sla_restart_jobmgr(self, failure_tester, maintenance): """ 1. Create a stateless job(instance_count=4) with host-limit-1 constraint and MaximumUnavailableInstances=1. Since there are only 3 UP hosts, one of the instances will not get placed (hence unavailable). 2. Start host maintenance on one of the hosts (say A). 3. Restart job manager. 4. Since one instance is already unavailable, no more instances should be killed due to host maintenance. Verify that host A does not transition to DOWN """ stateless_job = failure_tester.stateless_job() job_spec_dump = load_test_config('test_stateless_job_spec_sla.yaml') json_format.ParseDict(job_spec_dump, stateless_job.job_spec) stateless_job.job_spec.instance_count = 4 stateless_job.create() stateless_job.wait_for_all_pods_running(num_pods=3) # Pick a host that is UP and start maintenance on it test_host1 = get_host_in_state( host_pb2.HOST_STATE_UP, failure_tester.client) # update the client in maintenance fixture maintenance["update_client"](failure_tester.client) resp = maintenance["start"]([test_host1]) assert resp leader = failure_tester.fw.get_leader_info(failure_tester.jobmgr) assert leader assert 0 != failure_tester.fw.restart(failure_tester.jobmgr, "leader") failure_tester.wait_for_leader_change(failure_tester.jobmgr, leader) failure_tester.reset_client() stateless_job.client = failure_tester.client # update the client of maintainance maintenance["update_client"](failure_tester.client) try: wait_for_host_state(test_host1, host_pb2.HOST_STATE_DOWN) assert False, 'Host should not transition to DOWN' except: assert is_host_in_state(test_host1, host_pb2.HOST_STATE_DRAINING) assert len(stateless_job.query_pods( states=[pod_pb2.POD_STATE_RUNNING])) == 3