def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader): arv_docker_clear_cache() arvados_cwl.add_arv_hints() api = mock.MagicMock() api._rootDesc = get_rootDesc() runner = arvados_cwl.executor.ArvCwlExecutor(api) self.assertEqual(runner.work_api, 'containers') list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } runner.api.collections().list().execute.return_value = { "items": [{ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999993+99" }] } runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 runner.secret_store = cwltool.secrets.SecretStore() loadingContext, runtimeContext = self.helper(runner) runner.fs_access = runtimeContext.make_fs_access( runtimeContext.basedir) tool, metadata = loadingContext.loader.resolve_ref( "tests/wf/scatter2.cwl") metadata["cwlVersion"] = tool["cwlVersion"] mockc = mock.MagicMock() mockcollection.side_effect = lambda *args, **kwargs: CollectionMock( mockc, *args, **kwargs) mockcollectionreader().find.return_value = arvados.arvfile.ArvadosFile( mock.MagicMock(), "token.txt") arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext) arvtool.formatgraph = None it = arvtool.job({}, mock.MagicMock(), runtimeContext) next(it).run(runtimeContext) next(it).run(runtimeContext) with open("tests/wf/scatter2_subwf.cwl") as f: subwf = StripYAMLComments(f.read()).rstrip() runner.api.container_requests( ).create.assert_called_with(body=JsonDiffMatcher({ "command": [ "cwltool", "--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml" ], "container_image": "99999999999999999999999999999993+99", "cwd": "/var/spool/cwl", "environment": { "HOME": "/var/spool/cwl", "TMPDIR": "/tmp" }, "mounts": { "/keep/99999999999999999999999999999999+118": { "kind": "collection", "portable_data_hash": "99999999999999999999999999999999+118" }, "/tmp": { "capacity": 1073741824, "kind": "tmp" }, "/var/spool/cwl": { "capacity": 1073741824, "kind": "tmp" }, "/var/spool/cwl/cwl.input.yml": { "kind": "collection", "path": "cwl.input.yml", "portable_data_hash": "99999999999999999999999999999996+99" }, "/var/spool/cwl/workflow.cwl": { "kind": "collection", "path": "workflow.cwl", "portable_data_hash": "99999999999999999999999999999996+99" }, "stdout": { "kind": "file", "path": "/var/spool/cwl/cwl.output.json" } }, "name": "scatterstep", "output_name": "Output for step scatterstep", "output_path": "/var/spool/cwl", "output_ttl": 0, "priority": 500, "properties": {}, "runtime_constraints": { "ram": 1073741824, "vcpus": 1 }, "scheduling_parameters": {}, "secret_mounts": {}, "state": "Committed", "use_existing": True })) mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)]) mockc.open().__enter__().write.assert_has_calls([ mock.call('''{ "fileblub": { "basename": "token.txt", "class": "File", "location": "/keep/99999999999999999999999999999999+118/token.txt", "size": 0 }, "sleeptime": 5 }''') ])
def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader): arv_docker_clear_cache() arvados_cwl.add_arv_hints() api = mock.MagicMock() api._rootDesc = get_rootDesc() runner = arvados_cwl.executor.ArvCwlExecutor(api) self.assertEqual(runner.work_api, 'containers') list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]] runner.api.collections().get().execute.return_value = { "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999993+99" } runner.api.collections().list().execute.return_value = { "items": [{ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999993+99" }] } runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 runner.secret_store = cwltool.secrets.SecretStore() loadingContext, runtimeContext = self.helper(runner) runner.fs_access = runtimeContext.make_fs_access( runtimeContext.basedir) loadingContext.do_update = True tool, metadata = loadingContext.loader.resolve_ref( "tests/wf/echo-wf.cwl") mockcollection.side_effect = lambda *args, **kwargs: CollectionMock( mock.MagicMock(), *args, **kwargs) arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext) arvtool.formatgraph = None it = arvtool.job({}, mock.MagicMock(), runtimeContext) next(it).run(runtimeContext) next(it).run(runtimeContext) with open("tests/wf/echo-subwf.cwl") as f: subwf = StripYAMLComments(f.read()) runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher({ 'output_ttl': 0, 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'scheduling_parameters': {}, 'name': u'echo-subwf', 'secret_mounts': {}, 'runtime_constraints': { 'API': True, 'vcpus': 3, 'ram': 1073741824 }, 'properties': {}, 'priority': 500, 'mounts': { '/var/spool/cwl/cwl.input.yml': { 'portable_data_hash': '99999999999999999999999999999996+99', 'kind': 'collection', 'path': 'cwl.input.yml' }, '/var/spool/cwl/workflow.cwl': { 'portable_data_hash': '99999999999999999999999999999996+99', 'kind': 'collection', 'path': 'workflow.cwl' }, 'stdout': { 'path': '/var/spool/cwl/cwl.output.json', 'kind': 'file' }, '/tmp': { 'kind': 'tmp', 'capacity': 1073741824 }, '/var/spool/cwl': { 'kind': 'tmp', 'capacity': 3221225472 } }, 'state': 'Committed', 'output_path': '/var/spool/cwl', 'container_image': '99999999999999999999999999999993+99', 'command': [ u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml' ], 'use_existing': True, 'output_name': u'Output for step echo-subwf', 'cwd': '/var/spool/cwl' }))
def test_run(self, mockcollection): try: arvados_cwl.add_arv_hints() runner = arvados_cwl.ArvCwlRunner(mock.MagicMock()) runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") tool, metadata = document_loader.resolve_ref( "tests/wf/scatter2.cwl") metadata["cwlVersion"] = tool["cwlVersion"] mockcollection( ).portable_data_hash.return_value = "99999999999999999999999999999999+118" make_fs_access = functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) arvtool = arvados_cwl.ArvadosWorkflow( runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access, loader=document_loader, makeTool=runner.arv_make_tool, metadata=metadata) arvtool.formatgraph = None it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access) it.next().run() it.next().run() with open("tests/wf/scatter2_subwf.cwl") as f: subwf = f.read() mockcollection().open().__enter__().write.assert_has_calls( [mock.call(subwf)]) mockcollection().open().__enter__().write.assert_has_calls( [mock.call('{sleeptime: 5}')]) runner.api.jobs( ).create.assert_called_with(body={ 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6', 'repository': 'arvados', 'script_version': 'master', 'script': 'crunchrunner', 'script_parameters': { 'tasks': [{ 'task.env': { 'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)' }, 'task.vwd': { 'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl', 'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml' }, 'command': [ u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml' ], 'task.stdout': 'cwl.output.json' }] }, 'runtime_constraints': { 'min_scratch_mb_per_node': 2048, 'min_cores_per_node': 1, 'docker_image': 'arvados/jobs', 'min_ram_mb_per_node': 1024 }, 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz' }, filters= [['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], [ 'script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6' ], [ 'docker_image_locator', 'in docker', 'arvados/jobs' ]], find_or_create=True) except: logging.exception("")
def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader): arvados_cwl.add_arv_hints() api = mock.MagicMock() api._rootDesc = get_rootDesc() runner = arvados_cwl.ArvCwlRunner(api) self.assertEqual(runner.work_api, 'jobs') list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]] runner.api.collections().get().execute.return_vaulue = { "portable_data_hash": "99999999999999999999999999999993+99" } runner.api.collections().list().execute.return_vaulue = { "items": [{ "portable_data_hash": "99999999999999999999999999999993+99" }] } runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") make_fs_access = functools.partial( arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) document_loader.fetcher_constructor = functools.partial( arvados_cwl.CollectionFetcher, api_client=api, fs_access=make_fs_access("")) document_loader.fetcher = document_loader.fetcher_constructor( document_loader.cache, document_loader.session) document_loader.fetch_text = document_loader.fetcher.fetch_text document_loader.check_exists = document_loader.fetcher.check_exists tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl") metadata["cwlVersion"] = tool["cwlVersion"] mockcollection( ).portable_data_hash.return_value = "99999999999999999999999999999999+118" arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access, loader=document_loader, makeTool=runner.arv_make_tool, metadata=metadata) arvtool.formatgraph = None it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access) it.next().run() it.next().run() with open("tests/wf/scatter2_subwf.cwl") as f: subwf = StripYAMLComments(f.read()) runner.api.jobs().create.assert_called_with( body=JsonDiffMatcher( { 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d', 'repository': 'arvados', 'script_version': 'master', 'script': 'crunchrunner', 'script_parameters': { 'tasks': [{ 'task.env': { 'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)' }, 'task.vwd': { 'workflow.cwl': '$(task.keep)/99999999999999999999999999999999+118/workflow.cwl', 'cwl.input.yml': '$(task.keep)/99999999999999999999999999999999+118/cwl.input.yml' }, 'command': [ u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml' ], 'task.stdout': 'cwl.output.json' }] }, 'runtime_constraints': { 'min_scratch_mb_per_node': 2048, 'min_cores_per_node': 1, 'docker_image': 'arvados/jobs', 'min_ram_mb_per_node': 1024 }, 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz' }), filters=[['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], [ 'script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d' ], ['docker_image_locator', 'in docker', 'arvados/jobs']], find_or_create=True) mockcollection().open().__enter__().write.assert_has_calls( [mock.call(subwf)]) mockcollection().open().__enter__().write.assert_has_calls([ mock.call('''{ "fileblub": { "basename": "token.txt", "class": "File", "location": "/keep/99999999999999999999999999999999+118/token.txt" }, "sleeptime": 5 }''') ])
def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader): arvados_cwl.add_arv_hints() api = mock.MagicMock() api._rootDesc = get_rootDesc() runner = arvados_cwl.executor.ArvCwlExecutor(api) self.assertEqual(runner.work_api, 'jobs') list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]] runner.api.collections().get().execute.return_vaulue = { "portable_data_hash": "99999999999999999999999999999993+99" } runner.api.collections().list().execute.return_vaulue = { "items": [{ "portable_data_hash": "99999999999999999999999999999993+99" }] } runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 loadingContext, runtimeContext = self.helper(runner) tool, metadata = loadingContext.loader.resolve_ref( "tests/wf/echo-wf.cwl") metadata["cwlVersion"] = tool["cwlVersion"] mockcollection.side_effect = lambda *args, **kwargs: CollectionMock( mock.MagicMock(), *args, **kwargs) arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext) arvtool.formatgraph = None it = arvtool.job({}, mock.MagicMock(), runtimeContext) it.next().run(runtimeContext) it.next().run(runtimeContext) with open("tests/wf/echo-subwf.cwl") as f: subwf = StripYAMLComments(f.read()) runner.api.jobs().create.assert_called_with( body=JsonDiffMatcher( { 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d', 'repository': 'arvados', 'script_version': 'master', 'script': 'crunchrunner', 'script_parameters': { 'tasks': [{ 'task.env': { 'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)' }, 'task.vwd': { 'workflow.cwl': '$(task.keep)/99999999999999999999999999999996+99/workflow.cwl', 'cwl.input.yml': '$(task.keep)/99999999999999999999999999999996+99/cwl.input.yml' }, 'command': [ u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml' ], 'task.stdout': 'cwl.output.json' }] }, 'runtime_constraints': { 'min_scratch_mb_per_node': 4096, 'min_cores_per_node': 3, 'docker_image': 'arvados/jobs', 'min_ram_mb_per_node': 1024 }, 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz' }), filters=[['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], [ 'script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d' ], ['docker_image_locator', 'in docker', 'arvados/jobs']], find_or_create=True)