def test_long_running_job(): mock_client = create_mocked_client() job_name = "a_job" namespace = "a_namespace" a_job_metadata = V1ObjectMeta(name=job_name) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ a_job_is_launched_list ] # running job running_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=0)) completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ running_job, completed_job ] mock_client.wait_for_job_success(job_name, namespace) # slept once waiting for job to complete assert len(mock_client.sleeper.mock_calls) == 1
def test_wait_for_job_success(): mock_client = create_mocked_client() job_name = "a_job" namespace = "a_namespace" a_job_metadata = V1ObjectMeta(name=job_name) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ a_job_is_launched_list ] completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ completed_job ] mock_client.wait_for_job_success(job_name, namespace) # logger should not have been called assert not mock_client.logger.mock_calls # sleeper should not have been called assert not mock_client.sleeper.mock_calls
def test_wait_for_job_with_api_errors(): mock_client = create_mocked_client() job_name = "a_job" namespace = "a_namespace" a_job_metadata = V1ObjectMeta(name=job_name) not_launched_yet_list = V1JobList(items=[]) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), not_launched_yet_list, a_job_is_launched_list, ] completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ completed_job ] mock_client.wait_for_job_success(job_name, namespace) # 2 attempts with errors + 1 not launched + 1 launched assert len(mock_client.batch_api.list_namespaced_job.mock_calls) == 4
def test_wait_for_job_with_api_errors_retry_limit_exceeded(): mock_client = create_mocked_client() job_name = "a_job" a_job_metadata = V1ObjectMeta(name=job_name) not_launched_yet_list = V1JobList(items=[]) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), not_launched_yet_list, a_job_is_launched_list, ] completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ completed_job ] with pytest.raises(DagsterK8sAPIRetryLimitExceeded): mock_client.wait_for_job_success("a_job", "a_namespace") # 4 attempts with errors assert len(mock_client.batch_api.list_namespaced_job.mock_calls) == 4
def test_wait_for_job_success_with_api_errors(): mock_client = create_mocked_client() job_name = "a_job" namespace = "a_namespace" a_job_metadata = V1ObjectMeta(name=job_name) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ a_job_is_launched_list ] completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ kubernetes.client.rest.ApiException(status=503, reason="Service unavailable"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), completed_job, ] mock_client.wait_for_job_success(job_name, namespace) # logger should not have been called assert not mock_client.logger.mock_calls # sleeper should not have been called assert not mock_client.sleeper.mock_calls # 2 attempts with errors + 1 SUCCESS assert len( mock_client.batch_api.read_namespaced_job_status.mock_calls) == 3
def test_wait_for_job_not_launched(): mock_client = create_mocked_client() job_name = "a_job" namespace = "a_namespace" a_job_metadata = V1ObjectMeta(name=job_name) not_launched_yet_list = V1JobList(items=[]) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ not_launched_yet_list, a_job_is_launched_list, ] completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ completed_job ] mock_client.wait_for_job_success(job_name, namespace) assert_logger_calls(mock_client.logger, ['Job "a_job" not yet launched, waiting']) assert len(mock_client.sleeper.mock_calls) == 1
def test_wait_for_job_success_with_api_errors_retry_limit_exceeded(): mock_client = create_mocked_client() job_name = "a_job" a_job_metadata = V1ObjectMeta(name=job_name) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ a_job_is_launched_list ] completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ kubernetes.client.rest.ApiException(status=429, reason="Too many requests"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=429, reason="Too many requests"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), completed_job, ] with pytest.raises(DagsterK8sAPIRetryLimitExceeded) as exc_info: mock_client.wait_for_job_success("a_job", "a_namespace") assert "Retry limit of {limit} exceeded: " "Unexpected error encountered in Kubernetes API Client.".format( limit=3) in str(exc_info.value) # logger should not have been called assert not mock_client.logger.mock_calls # sleeper should not have been called assert not mock_client.sleeper.mock_calls # 2 retries due to errors + 1 SUCCESS assert len( mock_client.batch_api.read_namespaced_job_status.mock_calls) == 3
def test_job_failed(): mock_client = create_mocked_client() job_name = 'a_job' namespace = 'a_namespace' a_job_metadata = V1ObjectMeta(name=job_name) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ a_job_is_launched_list ] # failed job failed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=1, succeeded=0)) mock_client.batch_api.read_namespaced_job_status.side_effect = [failed_job] with pytest.raises(DagsterK8sError) as exc_info: mock_client.wait_for_job_success(job_name, namespace) assert 'Encountered failed job pods with status' in str(exc_info.value)
def test_wait_for_job_success_with_unrecoverable_api_errors(): mock_client = create_mocked_client() job_name = "a_job" a_job_metadata = V1ObjectMeta(name=job_name) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ a_job_is_launched_list ] completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=503, reason="Service unavailable"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), completed_job, ] with pytest.raises(DagsterK8sUnrecoverableAPIError) as exc_info: mock_client.wait_for_job_success("a_job", "a_namespace") assert "Unexpected error encountered in Kubernetes API Client." in str( exc_info.value) # logger should not have been called assert not mock_client.logger.mock_calls # sleeper should not have been called assert not mock_client.sleeper.mock_calls # 2 retries due to errors + 1 SUCCESS assert len( mock_client.batch_api.read_namespaced_job_status.mock_calls) == 2
def test_wait_for_job_success_with_api_errors_retry_limit_exceeded(): mock_client = create_mocked_client() job_name = "a_job" a_job_metadata = V1ObjectMeta(name=job_name) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ a_job_is_launched_list ] completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ kubernetes.client.rest.ApiException(status=500, reason="Internal server error"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=503, reason="Service unavailable"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), completed_job, ] with pytest.raises(DagsterK8sAPIRetryLimitExceeded): mock_client.wait_for_job_success("a_job", "a_namespace") # logger should not have been called assert not mock_client.logger.mock_calls # sleeper should not have been called assert not mock_client.sleeper.mock_calls # 4 attempts with errors assert len( mock_client.batch_api.read_namespaced_job_status.mock_calls) == 4
def test_wait_for_job_with_api_errors_retry_limit_exceeded(): mock_client = create_mocked_client() job_name = "a_job" a_job_metadata = V1ObjectMeta(name=job_name) not_launched_yet_list = V1JobList(items=[]) a_job_is_launched_list = V1JobList(items=[V1Job(metadata=a_job_metadata)]) mock_client.batch_api.list_namespaced_job.side_effect = [ kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), kubernetes.client.rest.ApiException(status=504, reason="Gateway Timeout"), not_launched_yet_list, a_job_is_launched_list, ] completed_job = V1Job(metadata=a_job_metadata, status=V1JobStatus(failed=0, succeeded=1)) mock_client.batch_api.read_namespaced_job_status.side_effect = [ completed_job ] with pytest.raises(DagsterK8sAPIRetryLimitExceeded) as exc_info: mock_client.wait_for_job_success("a_job", "a_namespace") assert "Retry limit of {limit} exceeded: " "Unexpected error encountered in Kubernetes API Client.".format( limit=3) in str(exc_info.value) # 2 retries due to errors + 1 not launched + 1 launched assert len(mock_client.batch_api.list_namespaced_job.mock_calls) == 3