def test_manual_rollback_increase_instances(stateless_job, in_place): stateless_job.create() stateless_job.wait_for_state(goal_state="RUNNING") update = StatelessUpdate( stateless_job, updated_job_file=UPDATE_STATELESS_JOB_ADD_INSTANCES_SPEC) update.create(in_place=in_place) update.wait_for_state(goal_state="SUCCEEDED") old_pod_infos = stateless_job.query_pods() old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() # reduce instance count and then roll it back update2 = StatelessUpdate( stateless_job, updated_job_file=UPDATE_STATELESS_JOB_UPDATE_REDUCE_INSTANCES_SPEC, ) update2.create(in_place=in_place) update3 = StatelessUpdate( stateless_job, updated_job_file=UPDATE_STATELESS_JOB_ADD_INSTANCES_SPEC) update3.create(in_place=in_place) update3.wait_for_state(goal_state="SUCCEEDED") new_pod_infos = stateless_job.query_pods() assert len(old_pod_infos) == len(new_pod_infos) new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test__auto_rollback_update_reduce_instances_with_bad_config( stateless_job, in_place): stateless_job.create() stateless_job.wait_for_state(goal_state="RUNNING") old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() job_spec_dump = load_test_config(UPDATE_STATELESS_JOB_BAD_SPEC) updated_job_spec = JobSpec() json_format.ParseDict(job_spec_dump, updated_job_spec) updated_job_spec.instance_count = stateless_job.job_spec.instance_count - 1 update = StatelessUpdate( stateless_job, updated_job_spec=updated_job_spec, roll_back_on_failure=True, max_failure_instances=1, max_instance_attempts=1, ) update.create(in_place=in_place) update.wait_for_state(goal_state="ROLLED_BACK") new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() # no instance should be removed assert (len( stateless_job.query_pods()) == stateless_job.job_spec.instance_count) assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test__auto_rollback_update_add_instances_with_bad_config(stateless_job): stateless_job.create() stateless_job.wait_for_state(goal_state='RUNNING') old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() job_spec_dump = load_test_config(UPDATE_STATELESS_JOB_BAD_SPEC) updated_job_spec = JobSpec() json_format.ParseDict(job_spec_dump, updated_job_spec) updated_job_spec.instance_count = \ stateless_job.job_spec.instance_count + 3 update = StatelessUpdate(stateless_job, updated_job_spec=updated_job_spec, roll_back_on_failure=True, max_failure_instances=1, max_instance_attempts=1) update.create() update.wait_for_state(goal_state='ROLLED_BACK') new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() # no instance should be added assert len(stateless_job.query_pods()) == \ stateless_job.job_spec.instance_count assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test__auto_rollback_update_with_failed_health_check(stateless_job): stateless_job.create() stateless_job.wait_for_state(goal_state='RUNNING') old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() update = StatelessUpdate( stateless_job, updated_job_file=UPDATE_STATELESS_JOB_BAD_HEALTH_CHECK_SPEC, roll_back_on_failure=True, max_failure_instances=1, max_instance_attempts=1) update.create() update.wait_for_state(goal_state='ROLLED_BACK') new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test__auto_rollback_update_with_bad_config(stateless_job, in_place): stateless_job.create() stateless_job.wait_for_state(goal_state="RUNNING") old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() update = StatelessUpdate( stateless_job, updated_job_file=UPDATE_STATELESS_JOB_BAD_SPEC, roll_back_on_failure=True, max_failure_instances=1, max_instance_attempts=1, ) update.create(in_place=in_place) update.wait_for_state(goal_state="ROLLED_BACK") new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test_manual_rollback_reduce_instances(stateless_job): stateless_job.create() stateless_job.wait_for_state(goal_state='RUNNING') old_pod_infos = stateless_job.query_pods() old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() update = StatelessUpdate( stateless_job, updated_job_file=UPDATE_STATELESS_JOB_ADD_INSTANCES_SPEC) update.create() # manually rollback the update update2 = StatelessUpdate( stateless_job, updated_job_file=UPDATE_STATELESS_JOB_UPDATE_REDUCE_INSTANCES_SPEC) update2.create() update2.wait_for_state(goal_state='SUCCEEDED') new_pod_infos = stateless_job.query_pods() assert len(old_pod_infos) == len(new_pod_infos) new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec() assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)