def test_manual_rollback_increase_instances(stateless_job, in_place):
stateless_job.create()
stateless_job.wait_for_state(goal_state="RUNNING")
update = StatelessUpdate(
stateless_job,
updated_job_file=UPDATE_STATELESS_JOB_ADD_INSTANCES_SPEC)
update.create(in_place=in_place)
update.wait_for_state(goal_state="SUCCEEDED")
old_pod_infos = stateless_job.query_pods()
old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
# reduce instance count and then roll it back
update2 = StatelessUpdate(
stateless_job,
updated_job_file=UPDATE_STATELESS_JOB_UPDATE_REDUCE_INSTANCES_SPEC,
)
update2.create(in_place=in_place)
update3 = StatelessUpdate(
stateless_job,
updated_job_file=UPDATE_STATELESS_JOB_ADD_INSTANCES_SPEC)
update3.create(in_place=in_place)
update3.wait_for_state(goal_state="SUCCEEDED")
new_pod_infos = stateless_job.query_pods()
assert len(old_pod_infos) == len(new_pod_infos)
new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test__auto_rollback_update_reduce_instances_with_bad_config(
stateless_job, in_place):
stateless_job.create()
stateless_job.wait_for_state(goal_state="RUNNING")
old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
job_spec_dump = load_test_config(UPDATE_STATELESS_JOB_BAD_SPEC)
updated_job_spec = JobSpec()
json_format.ParseDict(job_spec_dump, updated_job_spec)
updated_job_spec.instance_count = stateless_job.job_spec.instance_count - 1
update = StatelessUpdate(
stateless_job,
updated_job_spec=updated_job_spec,
roll_back_on_failure=True,
max_failure_instances=1,
max_instance_attempts=1,
)
update.create(in_place=in_place)
update.wait_for_state(goal_state="ROLLED_BACK")
new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
# no instance should be removed
assert (len(
stateless_job.query_pods()) == stateless_job.job_spec.instance_count)
assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test__auto_rollback_update_add_instances_with_bad_config(stateless_job):
stateless_job.create()
stateless_job.wait_for_state(goal_state='RUNNING')
old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
job_spec_dump = load_test_config(UPDATE_STATELESS_JOB_BAD_SPEC)
updated_job_spec = JobSpec()
json_format.ParseDict(job_spec_dump, updated_job_spec)
updated_job_spec.instance_count = \
stateless_job.job_spec.instance_count + 3
update = StatelessUpdate(stateless_job,
updated_job_spec=updated_job_spec,
roll_back_on_failure=True,
max_failure_instances=1,
max_instance_attempts=1)
update.create()
update.wait_for_state(goal_state='ROLLED_BACK')
new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
# no instance should be added
assert len(stateless_job.query_pods()) == \
stateless_job.job_spec.instance_count
assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test__auto_rollback_update_with_failed_health_check(stateless_job):
stateless_job.create()
stateless_job.wait_for_state(goal_state='RUNNING')
old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
update = StatelessUpdate(
stateless_job,
updated_job_file=UPDATE_STATELESS_JOB_BAD_HEALTH_CHECK_SPEC,
roll_back_on_failure=True,
max_failure_instances=1,
max_instance_attempts=1)
update.create()
update.wait_for_state(goal_state='ROLLED_BACK')
new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test__auto_rollback_update_with_bad_config(stateless_job, in_place):
stateless_job.create()
stateless_job.wait_for_state(goal_state="RUNNING")
old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
update = StatelessUpdate(
stateless_job,
updated_job_file=UPDATE_STATELESS_JOB_BAD_SPEC,
roll_back_on_failure=True,
max_failure_instances=1,
max_instance_attempts=1,
)
update.create(in_place=in_place)
update.wait_for_state(goal_state="ROLLED_BACK")
new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
def test_manual_rollback_reduce_instances(stateless_job):
stateless_job.create()
stateless_job.wait_for_state(goal_state='RUNNING')
old_pod_infos = stateless_job.query_pods()
old_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
update = StatelessUpdate(
stateless_job,
updated_job_file=UPDATE_STATELESS_JOB_ADD_INSTANCES_SPEC)
update.create()
# manually rollback the update
update2 = StatelessUpdate(
stateless_job,
updated_job_file=UPDATE_STATELESS_JOB_UPDATE_REDUCE_INSTANCES_SPEC)
update2.create()
update2.wait_for_state(goal_state='SUCCEEDED')
new_pod_infos = stateless_job.query_pods()
assert len(old_pod_infos) == len(new_pod_infos)
new_instance_zero_spec = stateless_job.get_pod(0).get_pod_spec()
assert_pod_spec_equal(old_instance_zero_spec, new_instance_zero_spec)
