def test_wait_for_termination_ready_then_terminate(): mock_client = create_mocked_client(timer=create_timing_out_timer( num_good_ticks=2)) single_not_ready_running_pod = _pod_list_for_container_status( _create_status( state=V1ContainerState(running=V1ContainerStateRunning()), ready=False)) single_pod_terminated_successful = _pod_list_for_container_status( _create_status(state=V1ContainerState( terminated=V1ContainerStateTerminated(exit_code=0)), ready=False)) mock_client.core_api.list_namespaced_pod.side_effect = [ single_not_ready_running_pod, single_pod_terminated_successful, ] pod_name = "a_pod" mock_client.wait_for_pod(pod_name=pod_name, namespace="namespace", wait_for_state=WaitForPodState.Terminated) assert_logger_calls( mock_client.logger, [ 'Waiting for pod "%s"' % pod_name, "Pod {pod_name} exitted successfully".format(pod_name=pod_name), ], ) # slept only once assert len(mock_client.sleeper.mock_calls) == 1
def test_wait_for_ready_but_terminated_unsuccessfully(): mock_client = create_mocked_client() single_not_ready_running_pod = _pod_list_for_container_status( _create_status( state=V1ContainerState(running=V1ContainerStateRunning()), ready=False)) single_pod_terminated_unsuccessful = _pod_list_for_container_status( _create_status( state=V1ContainerState(terminated=V1ContainerStateTerminated( exit_code=1, message="error_message")), ready=False, )) mock_client.core_api.list_namespaced_pod.side_effect = [ single_not_ready_running_pod, single_pod_terminated_unsuccessful, ] retrieve_pod_logs_mock = mock.MagicMock() retrieve_pod_logs_mock.side_effect = ["raw_logs_ret_val"] mock_client.retrieve_pod_logs = retrieve_pod_logs_mock pod_name = "a_pod" with pytest.raises(DagsterK8sError) as exc_info: mock_client.wait_for_pod(pod_name=pod_name, namespace="namespace") assert str(exc_info.value) == ( 'Pod did not exit successfully. Failed with message: "error_message" ' 'and pod logs: "raw_logs_ret_val"')
def test_waiting_for_pod_container_creation(): mock_client = create_mocked_client(timer=create_timing_out_timer( num_good_ticks=2)) single_waiting_pod = _pod_list_for_container_status( _create_status( state=V1ContainerState(waiting=V1ContainerStateWaiting( reason=KubernetesWaitingReasons.ContainerCreating)), ready=False, )) single_ready_running_pod = _pod_list_for_container_status( _ready_running_status()) mock_client.core_api.list_namespaced_pod.side_effect = [ single_waiting_pod, single_ready_running_pod, ] pod_name = "a_pod" mock_client.wait_for_pod(pod_name=pod_name, namespace="namespace") assert_logger_calls( mock_client.logger, [ 'Waiting for pod "%s"' % pod_name, "Waiting for container creation...", 'Pod "%s" is ready, done waiting' % pod_name, ], ) # slept only once assert len(mock_client.sleeper.mock_calls) == 1
def test_running_but_not_ready(): mock_client = create_mocked_client() single_not_ready_running_pod = _pod_list_for_container_status( _create_status( state=V1ContainerState(running=V1ContainerStateRunning()), ready=False)) single_ready_running_pod = _pod_list_for_container_status( _ready_running_status()) mock_client.core_api.list_namespaced_pod.side_effect = [ single_not_ready_running_pod, single_ready_running_pod, ] pod_name = "a_pod" mock_client.wait_for_pod(pod_name=pod_name, namespace="namespace") assert_logger_calls( mock_client.logger, [ 'Waiting for pod "%s"' % pod_name, 'Waiting for pod "%s" to become ready...' % pod_name, 'Pod "%s" is ready, done waiting' % pod_name, ], ) # slept only once assert len(mock_client.sleeper.mock_calls) == 1
def test_bad_waiting_state(): mock_client = create_mocked_client(timer=create_timing_out_timer( num_good_ticks=2)) single_waiting_pod = _pod_list_for_container_status( _create_status( state=V1ContainerState(waiting=V1ContainerStateWaiting( reason="InvalidReason")), ready=False, )) mock_client.core_api.list_namespaced_pod.side_effect = [single_waiting_pod] pod_name = "a_pod" with pytest.raises(DagsterK8sError) as exc_info: mock_client.wait_for_pod(pod_name=pod_name, namespace="namespace") assert str( exc_info.value ) == "Unknown issue: {'message': None, 'reason': 'InvalidReason'}"
def test_initial_timeout(): mock_client = create_mocked_client(timer=create_timing_out_timer( num_good_ticks=0)) not_ready_list = _pod_list_for_container_status( _create_status( state=V1ContainerState(running=V1ContainerStateRunning()), ready=False)) mock_client.core_api.list_namespaced_pod.side_effect = [not_ready_list] pod_name = "a_pod" with pytest.raises(DagsterK8sError) as exc_info: mock_client.wait_for_pod(pod_name=pod_name, namespace="namespace") # value of pod info is big blob of serialized dict info assert str(exc_info.value).startswith( "Timed out while waiting for pod to become ready with pod info:")
def test_wait_for_ready_but_terminated(): mock_client = create_mocked_client() single_pod_terminated_successful = _pod_list_for_container_status( _create_status(state=V1ContainerState( terminated=V1ContainerStateTerminated(exit_code=0)), ready=False)) mock_client.core_api.list_namespaced_pod.side_effect = [ single_pod_terminated_successful, ] pod_name = "a_pod" mock_client.wait_for_pod(pod_name=pod_name, namespace="namespace") assert_logger_calls( mock_client.logger, [ 'Waiting for pod "%s"' % pod_name, "Pod {pod_name} exitted successfully".format(pod_name=pod_name), ], )
def test_valid_failure_waiting_reasons(): mock_client = create_mocked_client() for reason in [ KubernetesWaitingReasons.ErrImagePull, KubernetesWaitingReasons.ImagePullBackOff, KubernetesWaitingReasons.CrashLoopBackOff, KubernetesWaitingReasons.RunContainerError, ]: single_waiting_pod_failure = _pod_list_for_container_status( _create_status( state=V1ContainerState(waiting=V1ContainerStateWaiting( reason=reason, message="bad things")), ready=False, )) mock_client.core_api.list_namespaced_pod.side_effect = [ single_waiting_pod_failure, ] pod_name = "a_pod" with pytest.raises(DagsterK8sError) as exc_info: mock_client.wait_for_pod(pod_name=pod_name, namespace="namespace") assert str( exc_info.value ) == 'Failed: Reason="{reason}" Message="bad things"'.format( reason=reason)
def _ready_running_status(): return _create_status( state=V1ContainerState(running=V1ContainerStateRunning()), ready=True)