def test_done(self, col): api = mock.MagicMock() runner = mock.MagicMock() runner.api = api runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.num_retries = 0 runner.ignore_docker_for_reuse = False col().open.return_value = [] api.collections().list().execute.side_effect = ({ "items": [] }, { "items": [{ "manifest_text": "XYZ" }] }) arvjob = arvados_cwl.ArvadosContainer(runner) arvjob.name = "testjob" arvjob.builder = mock.MagicMock() arvjob.output_callback = mock.MagicMock() arvjob.collect_outputs = mock.MagicMock() arvjob.successCodes = [0] arvjob.outdir = "/var/spool/cwl" arvjob.done({ "state": "Complete", "output": "99999999999999999999999999999993+99", "log": "99999999999999999999999999999994+99", "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "exit_code": 0 }) api.collections().list.assert_has_calls([ mock.call(), mock.call( filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'], [ 'portable_data_hash', '=', '99999999999999999999999999999993+99' ], ['name', '=', 'Output 9999999 of testjob']]), mock.call().execute(num_retries=0), mock.call(limit=1, filters=[[ 'portable_data_hash', '=', '99999999999999999999999999999993+99' ]], select=['manifest_text']), mock.call().execute(num_retries=0) ]) api.collections().create.assert_called_with( ensure_unique_name=True, body={ 'portable_data_hash': '99999999999999999999999999999993+99', 'manifest_text': 'XYZ', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'name': 'Output 9999999 of testjob' })
def test_child_failure(self, col, reader, gcc_mock, rts_mock): runner = self.setup_and_test_container_executor_and_logging(gcc_mock) gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"} self.assertTrue(gcc_mock.called) runner.num_retries = 0 runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.label = mock.MagicMock() runner.label.return_value = '[container testjob]' runner.api.containers().get().execute.return_value = { "state":"Complete", "output": "abc+123", "exit_code": 1, "log": "def+234" } col().open.return_value = [] loadingContext, runtimeContext = self.helper(runner) arvjob = arvados_cwl.ArvadosContainer(runner, runtimeContext, mock.MagicMock(), {}, None, [], [], "testjob") arvjob.output_callback = mock.MagicMock() arvjob.collect_outputs = mock.MagicMock() arvjob.successCodes = [0] arvjob.outdir = "/var/spool/cwl" arvjob.output_ttl = 3600 arvjob.collect_outputs.return_value = {"out": "stuff"} arvjob.done({ "state": "Final", "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1", "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "modified_at": "2017-05-26T12:01:22Z" }) rts_mock.assert_called_with( 'error', 'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:', ' ** log is empty **' ) arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
def test_done(self, col): api = mock.MagicMock() runner = mock.MagicMock() runner.api = api runner.num_retries = 0 runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api.containers().get().execute.return_value = {"state":"Complete", "output": "abc+123", "exit_code": 0} col().open.return_value = [] loadingContext, runtimeContext = self.helper(runner) arvjob = arvados_cwl.ArvadosContainer(runner, runtimeContext, mock.MagicMock(), {}, None, [], [], "testjob") arvjob.output_callback = mock.MagicMock() arvjob.collect_outputs = mock.MagicMock() arvjob.successCodes = [0] arvjob.outdir = "/var/spool/cwl" arvjob.output_ttl = 3600 arvjob.collect_outputs.return_value = {"out": "stuff"} arvjob.done({ "state": "Final", "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1", "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "modified_at": "2017-05-26T12:01:22Z" }) self.assertFalse(api.collections().create.called) self.assertFalse(runner.runtime_status_error.called) arvjob.collect_outputs.assert_called_with("keep:abc+123", 0) arvjob.output_callback.assert_called_with({"out": "stuff"}, "success") runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
def test_done(self, col): api = mock.MagicMock() runner = mock.MagicMock() runner.api = api runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.num_retries = 0 runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.api.containers().get().execute.return_value = { "state": "Complete", "output": "abc+123", "exit_code": 0 } col().open.return_value = [] arvjob = arvados_cwl.ArvadosContainer(runner) arvjob.name = "testjob" arvjob.builder = mock.MagicMock() arvjob.output_callback = mock.MagicMock() arvjob.collect_outputs = mock.MagicMock() arvjob.successCodes = [0] arvjob.outdir = "/var/spool/cwl" arvjob.output_ttl = 3600 arvjob.collect_outputs.return_value = {"out": "stuff"} arvjob.done({ "state": "Final", "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1", "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "modified_at": "2017-05-26T12:01:22Z" }) self.assertFalse(api.collections().create.called) arvjob.collect_outputs.assert_called_with("keep:abc+123") arvjob.output_callback.assert_called_with({"out": "stuff"}, "success") runner.add_intermediate_output.assert_called_with( "zzzzz-4zz18-zzzzzzzzzzzzzz2")
def test_done_use_existing_collection(self, col): api = mock.MagicMock() runner = mock.MagicMock() runner.api = api runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.num_retries = 0 col().open.return_value = [] api.collections().list().execute.side_effect = ({ "items": [{ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2" }] }, ) arvjob = arvados_cwl.ArvadosContainer(runner) arvjob.name = "testjob" arvjob.builder = mock.MagicMock() arvjob.output_callback = mock.MagicMock() arvjob.collect_outputs = mock.MagicMock() arvjob.successCodes = [0] arvjob.outdir = "/var/spool/cwl" arvjob.done({ "state": "Complete", "output": "99999999999999999999999999999993+99", "log": "99999999999999999999999999999994+99", "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "exit_code": 0 }) api.collections().list.assert_has_calls([ mock.call(), mock.call( filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'], [ 'portable_data_hash', '=', '99999999999999999999999999999993+99' ], ['name', '=', 'Output 9999999 of testjob']]), mock.call().execute(num_retries=0) ]) self.assertFalse(api.collections().create.called)
def test_child_failure(self, col, reader, gcc_mock): api = mock.MagicMock() api._rootDesc = copy.deepcopy(get_rootDesc()) del api._rootDesc.get('resources')['jobs']['methods']['create'] # Set up runner with mocked runtime_status_update() self.assertFalse(gcc_mock.called) runtime_status_update = mock.MagicMock() arvados_cwl.ArvCwlExecutor.runtime_status_update = runtime_status_update runner = arvados_cwl.ArvCwlExecutor(api) self.assertEqual(runner.work_api, 'containers') # Make sure ArvCwlExecutor thinks it's running inside a container so it # adds the logging handler that will call runtime_status_update() mock gcc_mock.return_value = {"uuid": "zzzzz-dz642-zzzzzzzzzzzzzzz"} self.assertTrue(gcc_mock.called) root_logger = logging.getLogger('') handlerClasses = [h.__class__ for h in root_logger.handlers] self.assertTrue( arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses) runner.num_retries = 0 runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.label = mock.MagicMock() runner.label.return_value = '[container testjob]' runner.api.containers().get().execute.return_value = { "state": "Complete", "output": "abc+123", "exit_code": 1, "log": "def+234" } col().open.return_value = [] loadingContext, runtimeContext = self.helper(runner) arvjob = arvados_cwl.ArvadosContainer(runner, runtimeContext, mock.MagicMock(), {}, None, [], [], "testjob") arvjob.output_callback = mock.MagicMock() arvjob.collect_outputs = mock.MagicMock() arvjob.successCodes = [0] arvjob.outdir = "/var/spool/cwl" arvjob.output_ttl = 3600 arvjob.collect_outputs.return_value = {"out": "stuff"} arvjob.done({ "state": "Final", "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1", "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "modified_at": "2017-05-26T12:01:22Z" }) runtime_status_update.assert_called_with( 'error', 'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:', ' ** log is empty **') arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")