def cleanup_stateless_jobs(timeout_secs=10):
"""
delete all service jobs from minicluster
"""
jobs = stateless_query_jobs()
# opportunistic delete for jobs, if not deleted within
# timeout period, it will get cleanup in next test run.
stateless_delete_jobs(jobs)
# Wait for job deletion to complete.
deadline = time.time() + timeout_secs
while time.time() < deadline:
try:
jobs = stateless_query_jobs()
if len(jobs) == 0:
return
time.sleep(2)
except grpc.RpcError as e:
# Catch "not-found" error here because QueryJobs endpoint does
# two db queries in sequence: "QueryJobs" and "GetUpdate".
# However, when we delete a job, updates are deleted first,
# there is a slight chance QueryJobs will fail to query the
# update, returning "not-found" error.
if e.code() == grpc.StatusCode.NOT_FOUND:
time.sleep(2)
continue
def clean_up():
# kill stateless jobs. This is needed since host draining
# is done in SLA aware manner for stateless jobs.
for j in stateless_query_jobs(client=client[0]):
j.stop()
if not draining_hosts:
return
for h in draining_hosts:
wait_for_host_state(h, host_pb2.HOST_STATE_DOWN)
stop(draining_hosts)