def test_long_running_job():
mock_client = create_mocked_client()
job_name = "a_job"
namespace = "a_namespace"
a_job_metadata = V1ObjectMeta(name=job_name)
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
a_job_is_launched_list
]
# running job
running_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=0))
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
running_job, completed_job
]
mock_client.wait_for_job_success(job_name, namespace)
# slept once waiting for job to complete
assert len(mock_client.sleeper.mock_calls) == 1
def test_wait_for_job_success():
mock_client = create_mocked_client()
job_name = "a_job"
namespace = "a_namespace"
a_job_metadata = V1ObjectMeta(name=job_name)
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
a_job_is_launched_list
]
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
completed_job
]
mock_client.wait_for_job_success(job_name, namespace)
# logger should not have been called
assert not mock_client.logger.mock_calls
# sleeper should not have been called
assert not mock_client.sleeper.mock_calls
def test_wait_for_job_with_api_errors():
mock_client = create_mocked_client()
job_name = "a_job"
namespace = "a_namespace"
a_job_metadata = V1ObjectMeta(name=job_name)
not_launched_yet_list = V1JobList(items=[])
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
not_launched_yet_list,
a_job_is_launched_list,
]
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
completed_job
]
mock_client.wait_for_job_success(job_name, namespace)
# 2 attempts with errors + 1 not launched + 1 launched
assert len(mock_client.batch_api.list_namespaced_job.mock_calls) == 4
def test_wait_for_job_with_api_errors_retry_limit_exceeded():
mock_client = create_mocked_client()
job_name = "a_job"
a_job_metadata = V1ObjectMeta(name=job_name)
not_launched_yet_list = V1JobList(items=[])
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
not_launched_yet_list,
a_job_is_launched_list,
]
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
completed_job
]
with pytest.raises(DagsterK8sAPIRetryLimitExceeded):
mock_client.wait_for_job_success("a_job", "a_namespace")
# 4 attempts with errors
assert len(mock_client.batch_api.list_namespaced_job.mock_calls) == 4
def test_wait_for_job_success_with_api_errors():
mock_client = create_mocked_client()
job_name = "a_job"
namespace = "a_namespace"
a_job_metadata = V1ObjectMeta(name=job_name)
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
a_job_is_launched_list
]
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
kubernetes.client.rest.ApiException(status=503,
reason="Service unavailable"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
completed_job,
]
mock_client.wait_for_job_success(job_name, namespace)
# logger should not have been called
assert not mock_client.logger.mock_calls
# sleeper should not have been called
assert not mock_client.sleeper.mock_calls
# 2 attempts with errors + 1 SUCCESS
assert len(
mock_client.batch_api.read_namespaced_job_status.mock_calls) == 3
def test_wait_for_job_not_launched():
mock_client = create_mocked_client()
job_name = "a_job"
namespace = "a_namespace"
a_job_metadata = V1ObjectMeta(name=job_name)
not_launched_yet_list = V1JobList(items=[])
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
not_launched_yet_list,
a_job_is_launched_list,
]
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
completed_job
]
mock_client.wait_for_job_success(job_name, namespace)
assert_logger_calls(mock_client.logger,
['Job "a_job" not yet launched, waiting'])
assert len(mock_client.sleeper.mock_calls) == 1
def test_wait_for_job_success_with_api_errors_retry_limit_exceeded():
mock_client = create_mocked_client()
job_name = "a_job"
a_job_metadata = V1ObjectMeta(name=job_name)
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
a_job_is_launched_list
]
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
kubernetes.client.rest.ApiException(status=429,
reason="Too many requests"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=429,
reason="Too many requests"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
completed_job,
]
with pytest.raises(DagsterK8sAPIRetryLimitExceeded) as exc_info:
mock_client.wait_for_job_success("a_job", "a_namespace")
assert "Retry limit of {limit} exceeded: " "Unexpected error encountered in Kubernetes API Client.".format(
limit=3) in str(exc_info.value)
# logger should not have been called
assert not mock_client.logger.mock_calls
# sleeper should not have been called
assert not mock_client.sleeper.mock_calls
# 2 retries due to errors + 1 SUCCESS
assert len(
mock_client.batch_api.read_namespaced_job_status.mock_calls) == 3
def test_job_failed():
mock_client = create_mocked_client()
job_name = 'a_job'
namespace = 'a_namespace'
a_job_metadata = V1ObjectMeta(name=job_name)
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
a_job_is_launched_list
]
# failed job
failed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=1, succeeded=0))
mock_client.batch_api.read_namespaced_job_status.side_effect = [failed_job]
with pytest.raises(DagsterK8sError) as exc_info:
mock_client.wait_for_job_success(job_name, namespace)
assert 'Encountered failed job pods with status' in str(exc_info.value)
def test_wait_for_job_success_with_unrecoverable_api_errors():
mock_client = create_mocked_client()
job_name = "a_job"
a_job_metadata = V1ObjectMeta(name=job_name)
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
a_job_is_launched_list
]
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=503,
reason="Service unavailable"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
completed_job,
]
with pytest.raises(DagsterK8sUnrecoverableAPIError) as exc_info:
mock_client.wait_for_job_success("a_job", "a_namespace")
assert "Unexpected error encountered in Kubernetes API Client." in str(
exc_info.value)
# logger should not have been called
assert not mock_client.logger.mock_calls
# sleeper should not have been called
assert not mock_client.sleeper.mock_calls
# 2 retries due to errors + 1 SUCCESS
assert len(
mock_client.batch_api.read_namespaced_job_status.mock_calls) == 2
def test_wait_for_job_success_with_api_errors_retry_limit_exceeded():
mock_client = create_mocked_client()
job_name = "a_job"
a_job_metadata = V1ObjectMeta(name=job_name)
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
a_job_is_launched_list
]
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
kubernetes.client.rest.ApiException(status=500,
reason="Internal server error"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=503,
reason="Service unavailable"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
completed_job,
]
with pytest.raises(DagsterK8sAPIRetryLimitExceeded):
mock_client.wait_for_job_success("a_job", "a_namespace")
# logger should not have been called
assert not mock_client.logger.mock_calls
# sleeper should not have been called
assert not mock_client.sleeper.mock_calls
# 4 attempts with errors
assert len(
mock_client.batch_api.read_namespaced_job_status.mock_calls) == 4
def test_wait_for_job_with_api_errors_retry_limit_exceeded():
mock_client = create_mocked_client()
job_name = "a_job"
a_job_metadata = V1ObjectMeta(name=job_name)
not_launched_yet_list = V1JobList(items=[])
a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)])
mock_client.batch_api.list_namespaced_job.side_effect = [
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
kubernetes.client.rest.ApiException(status=504,
reason="Gateway Timeout"),
not_launched_yet_list,
a_job_is_launched_list,
]
completed_job = V1Job(metadata=a_job_metadata,
status=V1JobStatus(failed=0, succeeded=1))
mock_client.batch_api.read_namespaced_job_status.side_effect = [
completed_job
]
with pytest.raises(DagsterK8sAPIRetryLimitExceeded) as exc_info:
mock_client.wait_for_job_success("a_job", "a_namespace")
assert "Retry limit of {limit} exceeded: " "Unexpected error encountered in Kubernetes API Client.".format(
limit=3) in str(exc_info.value)
# 2 retries due to errors + 1 not launched + 1 launched
assert len(mock_client.batch_api.list_namespaced_job.mock_calls) == 3