def test_delete_pod_name_raises(self, mock_get_namespace, mock_client): mock_client.rest.ApiException = Exception mock_client.CoreV1Api.return_value.delete_namespaced_pod.side_effect = ApiException kc = KubernetesClient() with self.assertRaisesRegex(CalrissianJobException, 'Error deleting pod named pod-123'): kc.delete_pod_name('pod-123')
def test_should_delete_pod_reads_env(self, mock_os, mock_get_namespace, mock_client): mock_os.getenv.return_value = 'NO' kc = KubernetesClient() self.assertFalse(kc.should_delete_pod()) self.assertEqual(mock_os.getenv.call_args, call('CALRISSIAN_DELETE_PODS', ''))
def test_is_terminated(self): self.assertFalse( KubernetesClient.state_is_terminated(self.running_state)) self.assertFalse( KubernetesClient.state_is_terminated(self.waiting_state)) self.assertTrue( KubernetesClient.state_is_terminated(self.terminated_state))
def __init__(self, *args, **kwargs): super(CalrissianCommandLineJob, self).__init__(*args, **kwargs) self.client = KubernetesClient() volume_builder = KubernetesVolumeBuilder() volume_builder.add_persistent_volume_entries_from_pod( self.client.get_current_pod()) self.volume_builder = volume_builder
def test_get_pod_for_name_not_found(self, mock_get_namespace, mock_client): mock_client.CoreV1Api.return_value.list_namespaced_pod.return_value = Mock( items=[]) kc = KubernetesClient() with self.assertRaisesRegex(CalrissianJobException, 'Unable to find pod with name somepod'): kc.get_pod_for_name('somepod')
def test_get_current_pod_missing_env_var(self, mock_os, mock_get_namespace, mock_client): mock_os.environ = {} kc = KubernetesClient() with self.assertRaisesRegex( CalrissianJobException, 'Missing required environment variable \$CALRISSIAN_POD_NAME'): kc.get_current_pod()
def test_get_pod_for_name_multiple_found(self, mock_get_namespace, mock_client): mock_client.CoreV1Api.return_value.list_namespaced_pod.return_value = Mock( items=['pod1', 'pod2']) kc = KubernetesClient() with self.assertRaisesRegex(CalrissianJobException, 'Multiple pods found with name somepod'): kc.get_pod_for_name('somepod')
def test_get_current_pod_missing_env_var(self, mock_os, mock_get_namespace, mock_client): mock_os.environ = {} kc = KubernetesClient() with self.assertRaises(CalrissianJobException) as raised_exception: kc.get_current_pod() self.assertEqual( str(raised_exception.exception), 'Missing required environment variable $CALRISSIAN_POD_NAME')
def test_delete_pod_name_ignores_404(self, mock_get_namespace, mock_client): mock_client.CoreV1Api.return_value.delete_namespaced_pod.side_effect = ApiException( status=404) kc = KubernetesClient() kc.delete_pod_name('pod-123') self.assertEqual( 'pod-123', mock_client.CoreV1Api.return_value. delete_namespaced_pod.call_args[0][0])
def test_get_current_pod_with_env_var(self, mock_os, mock_get_namespace, mock_client): mock_client.CoreV1Api.return_value.list_namespaced_pod.return_value = Mock( items=[{ 'name': 'mypod' }]) mock_os.environ = {'CALRISSIAN_POD_NAME': 'mypod'} kc = KubernetesClient() pod = kc.get_current_pod() self.assertEqual(pod, {'name': 'mypod'}) mock_client.CoreV1Api.return_value.list_namespaced_pod.assert_called_with( mock_get_namespace.return_value, field_selector='metadata.name=mypod')
def test_init(self, mock_get_namespace, mock_client): kc = KubernetesClient() self.assertEqual(kc.namespace, mock_get_namespace.return_value) self.assertEqual(kc.core_api_instance, mock_client.CoreV1Api.return_value) self.assertIsNone(kc.pod) self.assertIsNone(kc.completion_result)
def test_submit_pod(self, mock_podmonitor, mock_get_namespace, mock_client): mock_get_namespace.return_value = 'namespace' mock_create_namespaced_pod = Mock() mock_create_namespaced_pod.return_value = Mock(metadata=Mock( uid='123')) mock_client.CoreV1Api.return_value.create_namespaced_pod = mock_create_namespaced_pod kc = KubernetesClient() mock_body = Mock() kc.submit_pod(mock_body) self.assertEqual(kc.pod.metadata.uid, '123') self.assertEqual(mock_create_namespaced_pod.call_args, call('namespace', mock_body)) # This is to inspect `with PodMonitor() as monitor`: self.assertTrue( mock_podmonitor.return_value.__enter__.return_value.add.called)
def test_wait_finishes_when_pod_state_is_terminated( self, mock_cpu_memory, mock_podmonitor, mock_watch, mock_get_namespace, mock_client): mock_pod = create_autospec(V1Pod) mock_pod.status.container_statuses[0].state = Mock( running=None, waiting=None, terminated=Mock(exit_code=123)) mock_cpu_memory.return_value = ('1', '1Mi') self.setup_mock_watch(mock_watch, [mock_pod]) kc = KubernetesClient() kc._set_pod(Mock()) completion_result = kc.wait_for_completion() self.assertEqual(completion_result.exit_code, 123) self.assertTrue(mock_watch.Watch.return_value.stop.called) self.assertTrue( mock_client.CoreV1Api.return_value.delete_namespaced_pod.called) self.assertIsNone(kc.pod) # This is to inspect `with PodMonitor() as monitor`: self.assertTrue( mock_podmonitor.return_value.__enter__.return_value.remove.called)
def test_wait_checks_should_delete_when_pod_state_is_terminated( self, mock_cpu_memory, mock_should_delete_pod, mock_watch, mock_get_namespace, mock_client): mock_pod = create_autospec(V1Pod) mock_pod.status.container_statuses[0].state = Mock( running=None, waiting=None, terminated=Mock(exit_code=123)) mock_cpu_memory.return_value = ('1', '1Mi') mock_should_delete_pod.return_value = False self.setup_mock_watch(mock_watch, [mock_pod]) kc = KubernetesClient() kc._set_pod(Mock()) completion_result = kc.wait_for_completion() self.assertEqual(completion_result.exit_code, 123) self.assertEqual(completion_result.memory, '1Mi') self.assertEqual(completion_result.cpus, '1') self.assertTrue(mock_watch.Watch.return_value.stop.called) self.assertFalse( mock_client.CoreV1Api.return_value.delete_namespaced_pod.called) self.assertIsNone(kc.pod)
def test_follow_logs_streams_to_logging(self, mock_log, mock_get_namespace, mock_client): mock_get_namespace.return_value = 'logging-ns' mock_read = mock_client.CoreV1Api.return_value.read_namespaced_pod_log mock_read.return_value.stream.return_value = [b'line1\n', b'line2\n'] mock_pod = self.make_mock_pod('logging-pod-123') kc = KubernetesClient() kc._set_pod(mock_pod) mock_log.reset_mock( ) # log will have other calls before calling follow_logs() kc.follow_logs() self.assertTrue(mock_read.called) self.assertEqual( mock_read.call_args, call('logging-pod-123', 'logging-ns', follow=True, _preload_content=False)) self.assertEqual( mock_log.debug.mock_calls, [call('[logging-pod-123] line1'), call('[logging-pod-123] line2')]) self.assertEqual(mock_log.info.mock_calls, [ call('[logging-pod-123] follow_logs start'), call('[logging-pod-123] follow_logs end') ])
def test_raises_on_set_second_pod(self, mock_get_namespace, mock_client): kc = KubernetesClient() kc._set_pod(Mock()) with self.assertRaises(CalrissianJobException) as context: kc._set_pod(Mock()) self.assertIn('This client is already observing pod', str(context.exception))
def test_wait_skips_pod_when_status_is_none(self, mock_watch, mock_get_namespace, mock_client): mock_pod = Mock(status=Mock(container_statuses=None)) self.setup_mock_watch(mock_watch, [mock_pod]) kc = KubernetesClient() kc._set_pod(Mock()) kc.wait_for_completion() self.assertFalse(mock_watch.Watch.return_value.stop.called) self.assertFalse( mock_client.CoreV1Api.return_value.delete_namespaced_pod.called) self.assertIsNotNone(kc.pod)
def test_wait_raises_exception_when_state_is_unexpected( self, mock_watch, mock_get_namespace, mock_client): mock_pod = create_autospec(V1Pod) mock_pod.status.container_statuses[0].state = Mock(running=None, waiting=None, terminated=None) self.setup_mock_watch(mock_watch, [mock_pod]) kc = KubernetesClient() kc._set_pod(Mock()) with self.assertRaisesRegex(CalrissianJobException, 'Unexpected pod container status'): kc.wait_for_completion()
def test_wait_calls_watch_pod_with_pod_name_field_selector( self, mock_watch, mock_get_namespace, mock_client): self.setup_mock_watch(mock_watch) mock_pod = self.make_mock_pod('test123') kc = KubernetesClient() kc._set_pod(mock_pod) kc.wait_for_completion() mock_stream = mock_watch.Watch.return_value.stream self.assertEqual( mock_stream.call_args, call(kc.core_api_instance.list_namespaced_pod, kc.namespace, field_selector='metadata.name=test123'))
def test_wait_skips_pod_when_state_is_waiting(self, mock_watch, mock_get_namespace, mock_client): mock_pod = create_autospec(V1Pod) mock_pod.status.container_statuses[0].state = Mock(running=None, waiting=True, terminated=None) self.setup_mock_watch(mock_watch, [mock_pod]) kc = KubernetesClient() kc._set_pod(Mock()) kc.wait_for_completion() self.assertFalse(mock_watch.Watch.return_value.stop.called) self.assertFalse( mock_client.CoreV1Api.return_value.delete_namespaced_pod.called) self.assertIsNotNone(kc.pod)
def test_delete_pod_name_calls_api(self, mock_get_namespace, mock_client): kc = KubernetesClient() kc.delete_pod_name('pod-123') self.assertEqual( 'pod-123', mock_client.CoreV1Api.return_value. delete_namespaced_pod.call_args[0][0])
def test_raises_on_set_second_pod(self, mock_get_namespace, mock_client): kc = KubernetesClient() kc._set_pod(Mock()) with self.assertRaisesRegex(CalrissianJobException, 'This client is already observing pod'): kc._set_pod(Mock())
def test_get_pod_for_name_one_found(self, mock_get_namespace, mock_client): mock_client.CoreV1Api.return_value.list_namespaced_pod.return_value = Mock( items=['pod1']) kc = KubernetesClient() pod = kc.get_pod_for_name('somepod') self.assertEqual(pod, 'pod1')
def test_multiple_statuses_raises(self): self.assertEqual(len(self.multiple_statuses), 2) with self.assertRaisesRegex(CalrissianJobException, 'Expected 0 or 1 containers, found 2'): KubernetesClient.get_first_or_none(self.multiple_statuses)
def test_singular_status(self): self.assertEqual(len(self.singular_status), 1) self.assertIsNotNone( KubernetesClient.get_first_or_none(self.singular_status))
def test_none_statuses(self): self.assertIsNone( KubernetesClient.get_first_or_none(self.none_statuses)) self.assertIsNone( KubernetesClient.get_first_or_none(self.empty_list_statuses))
class CalrissianCommandLineJob(ContainerCommandLineJob): container_tmpdir = '/tmp' def __init__(self, *args, **kwargs): super(CalrissianCommandLineJob, self).__init__(*args, **kwargs) self.client = KubernetesClient() volume_builder = KubernetesVolumeBuilder() volume_builder.add_persistent_volume_entries_from_pod( self.client.get_current_pod()) self.volume_builder = volume_builder def make_tmpdir(self): # Doing this because cwltool.job does it if not os.path.exists(self.tmpdir): log.debug('os.makedirs({})'.format(self.tmpdir)) os.makedirs(self.tmpdir) def populate_env_vars(self): # cwltool DockerCommandLineJob always sets HOME to self.builder.outdir # https://github.com/common-workflow-language/cwltool/blob/1.0.20181201184214/cwltool/docker.py#L338 self.environment["HOME"] = self.builder.outdir # cwltool DockerCommandLineJob always sets TMPDIR to /tmp # https://github.com/common-workflow-language/cwltool/blob/1.0.20181201184214/cwltool/docker.py#L333 self.environment["TMPDIR"] = self.container_tmpdir def wait_for_kubernetes_pod(self): return self.client.wait_for_completion() def report(self, completion_result, disk_bytes): """ Convert the k8s-specific completion result into a report and submit it :param completion_result: calrissian.k8s.CompletionResult """ report = TimedResourceReport.create(self.name, completion_result, disk_bytes) Reporter.add_report(report) def finish(self, completion_result, runtimeContext): exit_code = completion_result.exit_code if exit_code in self.successCodes: status = "success" elif exit_code in self.temporaryFailCodes: status = "temporaryFail" elif exit_code in self.permanentFailCodes: status = "permanentFail" elif exit_code == 0: status = "success" else: status = "permanentFail" # collect_outputs (and collect_output) is defined in command_line_tool outputs = self.collect_outputs(self.outdir, exit_code) disk_bytes = total_size(outputs) self.report(completion_result, disk_bytes) # Invoke the callback with a lock with runtimeContext.workflow_eval_lock: self.output_callback(outputs, status) # Cleanup our stagedir and tmp if self.stagedir is not None and os.path.exists(self.stagedir): log.debug('shutil.rmtree({}, {})'.format(self.stagedir, True)) shutil.rmtree(self.stagedir, True) if runtimeContext.rm_tmpdir: log.debug('shutil.rmtree({}, {})'.format(self.tmpdir, True)) shutil.rmtree(self.tmpdir, True) # Dictionary of supported features. # Not yet complete, only checks features of DockerRequirement supported_features = {'DockerRequirement': ['class', 'dockerPull']} def check_requirements(self): for feature in self.supported_features: requirement, is_required = self.get_requirement(feature) if requirement and is_required: for field in requirement: if not field in self.supported_features[feature]: raise UnsupportedRequirement( 'Error: feature {}.{} is not supported'.format( feature, field)) def _get_container_image(self): docker_requirement, _ = self.get_requirement('DockerRequirement') if docker_requirement: container_image = docker_requirement['dockerPull'] else: # No dockerRequirement, use the default container container_image = self.builder.find_default_container() if not container_image: raise CalrissianCommandLineJobException( 'Unable to create Job - Please ensure tool has a DockerRequirement with dockerPull or specify a default_container' ) return container_image def quoted_command_line(self): return quoted_arg_list(self.command_line) def get_pod_labels(self, runtimeContext): if runtimeContext.pod_labels: return read_yaml(runtimeContext.pod_labels) else: return {} def create_kubernetes_runtime(self, runtimeContext): # In cwltool, the runtime list starts as something like ['docker','run'] and these various builder methods # append to that list with docker (or singularity) options like volume mount paths # As we build up kubernetes, these aren't really used this way so we leave it empty runtime = [] # Append volume for outdir self._add_volume_binding(os.path.realpath(self.outdir), self.builder.outdir, writable=True) # Use a kubernetes emptyDir: {} volume for /tmp # Note that below add_volumes() may result in other temporary files being mounted # from the calrissian host's tmpdir prefix into an absolute container path, but this will # not conflict with '/tmp' as an emptyDir self._add_emptydir_volume_and_binding('tmpdir', self.container_tmpdir) # Call the ContainerCommandLineJob add_volumes method self.add_volumes(self.pathmapper, runtime, tmpdir_prefix=runtimeContext.tmpdir_prefix, secret_store=runtimeContext.secret_store, any_path_okay=True) if self.generatemapper is not None: # Seems to be true if docker is a hard requirement # This evaluates to true if docker_is_required is true # Used only for generatemapper add volumes any_path_okay = self.builder.get_requirement( "DockerRequirement")[1] or False self.add_volumes(self.generatemapper, runtime, tmpdir_prefix=runtimeContext.tmpdir_prefix, secret_store=runtimeContext.secret_store, any_path_okay=any_path_okay) pod_labels = self.get_pod_labels(runtimeContext) k8s_builder = KubernetesPodBuilder( self.name, self._get_container_image(), self.environment, self.volume_builder.volume_mounts, self.volume_builder.volumes, self.quoted_command_line(), self.stdout, self.stderr, self.stdin, self.builder.resources, pod_labels, ) built = k8s_builder.build() log.debug('{}\n{}{}\n'.format('-' * 80, yaml.dump(built), '-' * 80)) # Report an error if anything was added to the runtime list if runtime: log.error( 'Runtime list is not empty. k8s does not use that, so you should see who put something there:\n{}' .format(' '.join(runtime))) return built def execute_kubernetes_pod(self, pod): self.client.submit_pod(pod) def _add_emptydir_volume_and_binding(self, name, target): self.volume_builder.add_emptydir_volume(name) self.volume_builder.add_emptydir_volume_binding(name, target) def _add_volume_binding(self, source, target, writable=False): self.volume_builder.add_volume_binding(source, target, writable) # Below are concrete implementations of methods called by add_volumes # They are based on https://github.com/common-workflow-language/cwltool/blob/1.0.20181201184214/cwltool/docker.py # But the key difference is that docker is invoked via command-line, so the ones in docker.py append to # a runtime list. Here, we instead call self._add_volume_binding() def add_file_or_directory_volume( self, runtime, # type: List[Text] volume, # type: MapperEnt host_outdir_tgt # type: Optional[Text] ): """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): self._add_volume_binding( volume.resolved, volume.target) # this one defaults to read_only def add_writable_file_volume( self, runtime, # type: List[Text] volume, # type: MapperEnt host_outdir_tgt, # type: Optional[Text] tmpdir_prefix # type: Text ): """Append a writable file mapping to the runtime option list.""" if self.inplace_update: self._add_volume_binding(volume.resolved, volume.target, writable=True) else: if host_outdir_tgt: # shortcut, just copy to the output directory # which is already going to be mounted log.debug('shutil.copy({}, {})'.format(volume.resolved, host_outdir_tgt)) shutil.copy(volume.resolved, host_outdir_tgt) else: log.debug('tempfile.mkdtemp(dir={})'.format(self.tmpdir)) tmpdir = tempfile.mkdtemp(dir=self.tmpdir) file_copy = os.path.join(tmpdir, os.path.basename(volume.resolved)) log.debug('shutil.copy({}, {})'.format(volume.resolved, file_copy)) shutil.copy(volume.resolved, file_copy) self._add_volume_binding(file_copy, volume.target, writable=True) ensure_writable(host_outdir_tgt or file_copy) def add_writable_directory_volume( self, runtime, # type: List[Text] volume, # type: MapperEnt host_outdir_tgt, # type: Optional[Text] tmpdir_prefix # type: Text ): """Append a writable directory mapping to the runtime option list.""" if volume.resolved.startswith("_:"): # Synthetic directory that needs creating first if not host_outdir_tgt: log.debug('tempfile.mkdtemp(dir={})'.format(self.tmpdir)) new_dir = os.path.join(tempfile.mkdtemp(dir=self.tmpdir), os.path.basename(volume.target)) self._add_volume_binding(new_dir, volume.target, writable=True) elif not os.path.exists(host_outdir_tgt): log.debug('os.makedirs({}, 0o0755)'.format(host_outdir_tgt)) os.makedirs(host_outdir_tgt, 0o0755) else: if self.inplace_update: self._add_volume_binding(volume.resolved, volume.target, writable=True) else: if not host_outdir_tgt: log.debug('tempfile.mkdtemp(dir={})'.format(self.tmpdir)) tmpdir = tempfile.mkdtemp(dir=self.tmpdir) new_dir = os.path.join(tmpdir, os.path.basename(volume.resolved)) log.debug('shutil.copytree({}, {})'.format( volume.resolved, new_dir)) shutil.copytree(volume.resolved, new_dir) self._add_volume_binding(new_dir, volume.target, writable=True) else: log.debug('shutil.copytree({}, {})'.format( volume.resolved, host_outdir_tgt)) shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) def run(self, runtimeContext, tmpdir_lock=None): self.check_requirements() if tmpdir_lock: with tmpdir_lock: self.make_tmpdir() else: self.make_tmpdir() self.populate_env_vars() self._setup(runtimeContext) pod = self.create_kubernetes_runtime( runtimeContext) # analogous to create_runtime() self.execute_kubernetes_pod(pod) # analogous to _execute() completion_result = self.wait_for_kubernetes_pod() self.finish(completion_result, runtimeContext) # Below are concrete implementations of the remaining abstract methods in ContainerCommandLineJob # They are not implemented and not expected to be called, so they all raise NotImplementedError def get_from_requirements( self, r, # type: Dict[Text, Text] pull_image, # type: bool force_pull=False, # type: bool tmp_outdir_prefix=DEFAULT_TMP_PREFIX # type: Text ): raise NotImplementedError('get_from_requirements') def create_runtime( self, env, # type: MutableMapping[Text, Text] runtime_context # type: RuntimeContext ): # expected to return runtime list and cid string raise NotImplementedError('create_runtime') def append_volume(self, runtime, source, target, writable=False): """Add volume binding to the arguments list. This is called by the base class for file literals after they've been created. We already have a similar function, so we just call that. """ self._add_volume_binding(source, target, writable)
def test_multiple_statuses_raises(self): self.assertEqual(len(self.multiple_statuses), 2) with self.assertRaises(CalrissianJobException) as context: KubernetesClient.get_first_or_none(self.multiple_statuses) self.assertIn('Expected 0 or 1 containers, found 2', str(context.exception))