def test_run(self, keepdocker): for enable_reuse in (True, False): arv_docker_clear_cache() runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99"} document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") tool = cmap({ "inputs": [], "outputs": [], "baseCommand": "ls", "arguments": [{"valueFrom": "$(runtime.outdir)"}] }) make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse), make_fs_access=make_fs_access, tmpdir="/tmp"): j.run(enable_reuse=enable_reuse) runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher({ 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_run_'+str(enable_reuse), 'runtime_constraints': { 'vcpus': 1, 'ram': 1073741824 }, 'use_existing': enable_reuse, 'priority': 1, 'mounts': { '/tmp': {'kind': 'tmp', "capacity": 1073741824 }, '/var/spool/cwl': {'kind': 'tmp', "capacity": 1073741824 } }, 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, 'container_image': 'arvados/jobs', 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {}, }))
def test_resource_requirements(self): runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False tool = { "inputs": [], "outputs": [], "hints": [{ "class": "ResourceRequirement", "coresMin": 3, "ramMin": 3000, "tmpdirMin": 4000 }], "baseCommand": "ls" } arvtool = arvados_cwl.ArvadosCommandTool(runner, tool) arvtool.formatgraph = None for j in arvtool.job({}, "", mock.MagicMock()): j.run() runner.api.jobs().create.assert_called_with( body={ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'runtime_constraints': {}, 'script_parameters': { 'tasks': [{ 'task.env': { 'TMPDIR': '$(task.tmpdir)' }, 'command': ['ls'] }] }, 'script_version': 'master', 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6', 'repository': 'arvados', 'script': 'crunchrunner', 'runtime_constraints': { 'docker_image': 'arvados/jobs', 'min_cores_per_node': 3, 'min_ram_mb_per_node': 3000, 'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize } }, find_or_create=True, filters=[['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], [ 'script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6' ], ['docker_image_locator', 'in docker', 'arvados/jobs']])
def test_timelimit(self, keepdocker): arv_docker_clear_cache() runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } tool = cmap({ "inputs": [], "outputs": [], "baseCommand": "ls", "arguments": [{ "valueFrom": "$(runtime.outdir)" }], "id": "#", "class": "CommandLineTool", "hints": [{ "class": "http://commonwl.org/cwltool#TimeLimit", "timelimit": 42 }] }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_timelimit" arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): j.run(runtimeContext) _, kwargs = runner.api.container_requests().create.call_args self.assertEqual( 42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
def test_initial_work_dir(self, collection_mock, keepdocker): arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } sourcemock = mock.MagicMock() def get_collection_mock(p): if "/" in p: return (sourcemock, p.split("/", 1)[1]) else: return (sourcemock, "") runner.fs_access.get_collection.side_effect = get_collection_mock vwdmock = mock.MagicMock() collection_mock.side_effect = lambda *args, **kwargs: CollectionMock( vwdmock, *args, **kwargs) tool = cmap({ "inputs": [], "outputs": [], "hints": [{ "class": "InitialWorkDirRequirement", "listing": [{ "class": "File", "basename": "foo", "location": "keep:99999999999999999999999999999995+99/bar" }, { "class": "Directory", "basename": "foo2", "location": "keep:99999999999999999999999999999995+99" }, { "class": "File", "basename": "filename", "location": "keep:99999999999999999999999999999995+99/baz/filename" }, { "class": "Directory", "basename": "subdir", "location": "keep:99999999999999999999999999999995+99/subdir" }] }], "baseCommand": "ls", "id": "#", "class": "CommandLineTool" }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_initial_work_dir" arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): j.run(runtimeContext) call_args, call_kwargs = runner.api.container_requests( ).create.call_args vwdmock.copy.assert_has_calls( [mock.call('bar', 'foo', source_collection=sourcemock)]) vwdmock.copy.assert_has_calls( [mock.call('.', 'foo2', source_collection=sourcemock)]) vwdmock.copy.assert_has_calls([ mock.call('baz/filename', 'filename', source_collection=sourcemock) ]) vwdmock.copy.assert_has_calls( [mock.call('subdir', 'subdir', source_collection=sourcemock)]) call_body_expected = { 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_initial_work_dir', 'runtime_constraints': { 'vcpus': 1, 'ram': 1073741824 }, 'use_existing': True, 'priority': 500, 'mounts': { '/tmp': { 'kind': 'tmp', "capacity": 1073741824 }, '/var/spool/cwl': { 'kind': 'tmp', "capacity": 1073741824 }, '/var/spool/cwl/foo': { 'kind': 'collection', 'path': 'foo', 'portable_data_hash': '99999999999999999999999999999996+99' }, '/var/spool/cwl/foo2': { 'kind': 'collection', 'path': 'foo2', 'portable_data_hash': '99999999999999999999999999999996+99' }, '/var/spool/cwl/filename': { 'kind': 'collection', 'path': 'filename', 'portable_data_hash': '99999999999999999999999999999996+99' }, '/var/spool/cwl/subdir': { 'kind': 'collection', 'path': 'subdir', 'portable_data_hash': '99999999999999999999999999999996+99' } }, 'state': 'Committed', 'output_name': 'Output for step test_initial_work_dir', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, 'container_image': '99999999999999999999999999999993+99', 'command': ['ls'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {}, 'secret_mounts': {} } call_body = call_kwargs.get('body', None) self.assertNotEqual(None, call_body) for key in call_body: self.assertEqual(call_body_expected.get(key), call_body.get(key))
def test_resource_requirements(self, keepdocker): arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 3600 runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } tool = cmap({ "inputs": [], "outputs": [], "hints": [{ "class": "ResourceRequirement", "coresMin": 3, "ramMin": 3000, "tmpdirMin": 4000, "outdirMin": 5000 }, { "class": "http://arvados.org/cwl#RuntimeConstraints", "keep_cache": 512 }, { "class": "http://arvados.org/cwl#APIRequirement", }, { "class": "http://arvados.org/cwl#PartitionRequirement", "partition": "blurb" }, { "class": "http://arvados.org/cwl#IntermediateOutput", "outputTTL": 7200 }, { "class": "http://arvados.org/cwl#ReuseRequirement", "enableReuse": False }], "baseCommand": "ls", "id": "#", "class": "CommandLineTool" }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_resource_requirements" arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): j.run(runtimeContext) call_args, call_kwargs = runner.api.container_requests( ).create.call_args call_body_expected = { 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_resource_requirements', 'runtime_constraints': { 'vcpus': 3, 'ram': 3145728000, 'keep_cache_ram': 536870912, 'API': True }, 'use_existing': False, 'priority': 500, 'mounts': { '/tmp': { 'kind': 'tmp', "capacity": 4194304000 }, '/var/spool/cwl': { 'kind': 'tmp', "capacity": 5242880000 } }, 'state': 'Committed', 'output_name': 'Output for step test_resource_requirements', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 7200, 'container_image': '99999999999999999999999999999993+99', 'command': ['ls'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': { 'partitions': ['blurb'] }, 'properties': {}, 'secret_mounts': {} } call_body = call_kwargs.get('body', None) self.assertNotEqual(None, call_body) for key in call_body: self.assertEqual(call_body_expected.get(key), call_body.get(key))
def test_run(self, keepdocker): for enable_reuse in (True, False): arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } tool = cmap({ "inputs": [], "outputs": [], "baseCommand": "ls", "arguments": [{ "valueFrom": "$(runtime.outdir)" }], "id": "#", "class": "CommandLineTool" }) loadingContext, runtimeContext = self.helper(runner, enable_reuse) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): j.run(runtimeContext) runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher({ 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_run_' + str(enable_reuse), 'runtime_constraints': { 'vcpus': 1, 'ram': 1073741824 }, 'use_existing': enable_reuse, 'priority': 500, 'mounts': { '/tmp': { 'kind': 'tmp', "capacity": 1073741824 }, '/var/spool/cwl': { 'kind': 'tmp', "capacity": 1073741824 } }, 'state': 'Committed', 'output_name': 'Output for step test_run_' + str(enable_reuse), 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, 'container_image': '99999999999999999999999999999993+99', 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {}, 'secret_mounts': {} }))
def test_run(self, list_images_in_arv): for enable_reuse in (True, False): runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } # Simulate reused job from another project so that we can check is a can_read # link is added. runner.api.jobs().create().execute.return_value = { 'state': 'Complete' if enable_reuse else 'Queued', 'owner_uuid': 'zzzzz-tpzed-yyyyyyyyyyyyyyy' if enable_reuse else 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'uuid': 'zzzzz-819sb-yyyyyyyyyyyyyyy', 'output': None, } tool = cmap({ "inputs": [], "outputs": [], "baseCommand": "ls", "arguments": [{ "valueFrom": "$(runtime.outdir)" }] }) make_fs_access = functools.partial( arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache( runner.api, None, 0)) arvtool = arvados_cwl.ArvadosCommandTool( runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access): j.run(enable_reuse=enable_reuse) runner.api.jobs().create.assert_called_with( body=JsonDiffMatcher({ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'runtime_constraints': {}, 'script_parameters': { 'tasks': [{ 'task.env': { 'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)' }, 'command': ['ls', '$(task.outdir)'] }], }, 'script_version': 'master', 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d', 'repository': 'arvados', 'script': 'crunchrunner', 'runtime_constraints': { 'docker_image': 'arvados/jobs', 'min_cores_per_node': 1, 'min_ram_mb_per_node': 1024, 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize } }), find_or_create=enable_reuse, filters=[['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], [ 'script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d' ], [ 'docker_image_locator', 'in docker', 'arvados/jobs' ]]) if enable_reuse: runner.api.links().create.assert_called_with( body=JsonDiffMatcher( { 'link_class': 'permission', 'name': 'can_read', "tail_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", "head_uuid": "zzzzz-819sb-yyyyyyyyyyyyyyy", })) # Simulate an API excepction when trying to create a # sharing link on the job runner.api.links().create.side_effect = ApiError( mock.MagicMock(return_value={'status': 403}), 'Permission denied') j.run(enable_reuse=enable_reuse) j.output_callback.assert_called_with({}, 'success') else: assert not runner.api.links().create.called
def test_resource_requirements(self, list_images_in_arv): runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 arvados_cwl.add_arv_hints() list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]] runner.api.collections().get().execute.return_vaulue = { "portable_data_hash": "99999999999999999999999999999993+99" } document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") tool = { "inputs": [], "outputs": [], "hints": [{ "class": "ResourceRequirement", "coresMin": 3, "ramMin": 3000, "tmpdirMin": 4000 }, { "class": "http://arvados.org/cwl#RuntimeConstraints", "keep_cache": 512, "outputDirType": "keep_output_dir" }, { "class": "http://arvados.org/cwl#APIRequirement", }, { "class": "http://arvados.org/cwl#ReuseRequirement", "enableReuse": False }], "baseCommand": "ls" } make_fs_access = functools.partial( arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names, make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access): j.run(enable_reuse=True) runner.api.jobs().create.assert_called_with( body=JsonDiffMatcher({ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'runtime_constraints': {}, 'script_parameters': { 'tasks': [{ 'task.env': { 'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)' }, 'task.keepTmpOutput': True, 'command': ['ls'] }] }, 'script_version': 'master', 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d', 'repository': 'arvados', 'script': 'crunchrunner', 'runtime_constraints': { 'docker_image': 'arvados/jobs', 'min_cores_per_node': 3, 'min_ram_mb_per_node': 3512, # ramMin + keep_cache 'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize 'keep_cache_mb_per_task': 512 } }), find_or_create=False, filters=[['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], [ 'script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d' ], ['docker_image_locator', 'in docker', 'arvados/jobs']])
def test_resource_requirements(self): runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 arvados_cwl.add_arv_hints() document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") tool = { "inputs": [], "outputs": [], "hints": [{ "class": "ResourceRequirement", "coresMin": 3, "ramMin": 3000, "tmpdirMin": 4000 }, { "class": "http://arvados.org/cwl#RuntimeConstraints", "keep_cache": 512, "outputDirType": "keep_output_dir" }, { "class": "http://arvados.org/cwl#APIRequirement", }], "baseCommand": "ls" } make_fs_access = functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names, make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access): j.run() runner.api.jobs().create.assert_called_with( body={ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'runtime_constraints': {}, 'script_parameters': { 'tasks': [{ 'task.env': { 'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)' }, 'task.keepTmpOutput': True, 'command': ['ls'] }] }, 'script_version': 'master', 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6', 'repository': 'arvados', 'script': 'crunchrunner', 'runtime_constraints': { 'docker_image': 'arvados/jobs', 'min_cores_per_node': 3, 'min_ram_mb_per_node': 3000, 'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize 'keep_cache_mb_per_task': 512 } }, find_or_create=True, filters=[['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], [ 'script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6' ], ['docker_image_locator', 'in docker', 'arvados/jobs']])
def test_resource_requirements(self, keepdocker): runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "draft-3") keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } tool = { "inputs": [], "outputs": [], "hints": [{ "class": "ResourceRequirement", "coresMin": 3, "ramMin": 3000, "tmpdirMin": 4000 }], "baseCommand": "ls" } make_fs_access = functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, make_fs_access=make_fs_access) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements", make_fs_access=make_fs_access, tmpdir="/tmp"): j.run() runner.api.container_requests().create.assert_called_with( body={ 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_resource_requirements', 'runtime_constraints': { 'vcpus': 3, 'ram': 3145728000 }, 'priority': 1, 'mounts': { '/var/spool/cwl': { 'kind': 'tmp' } }, 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'container_image': '99999999999999999999999999999993+99', 'command': ['ls'], 'cwd': '/var/spool/cwl' })
def test_redirects(self, keepdocker): arv_docker_clear_cache() runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") tool = cmap({ "inputs": [], "outputs": [], "baseCommand": "ls", "stdout": "stdout.txt", "stderr": "stderr.txt", "stdin": "/keep/99999999999999999999999999999996+99/file.txt", "arguments": [{ "valueFrom": "$(runtime.outdir)" }], "id": "#", "class": "CommandLineTool" }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_run_redirect" arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): j.run(runtimeContext) runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher({ 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_run_redirect', 'runtime_constraints': { 'vcpus': 1, 'ram': 1073741824 }, 'use_existing': True, 'priority': 500, 'mounts': { '/tmp': { 'kind': 'tmp', "capacity": 1073741824 }, '/var/spool/cwl': { 'kind': 'tmp', "capacity": 1073741824 }, "stderr": { "kind": "file", "path": "/var/spool/cwl/stderr.txt" }, "stdin": { "kind": "collection", "path": "file.txt", "portable_data_hash": "99999999999999999999999999999996+99" }, "stdout": { "kind": "file", "path": "/var/spool/cwl/stdout.txt" }, }, 'state': 'Committed', "output_name": "Output for step test_run_redirect", 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, 'container_image': 'arvados/jobs', 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {}, 'secret_mounts': {} }))
def test_initial_work_dir(self, collection_mock, keepdocker): arv_docker_clear_cache() runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } sourcemock = mock.MagicMock() def get_collection_mock(p): if "/" in p: return (sourcemock, p.split("/", 1)[1]) else: return (sourcemock, "") runner.fs_access.get_collection.side_effect = get_collection_mock vwdmock = mock.MagicMock() collection_mock.return_value = vwdmock vwdmock.portable_data_hash.return_value = "99999999999999999999999999999996+99" tool = cmap({ "inputs": [], "outputs": [], "hints": [{ "class": "InitialWorkDirRequirement", "listing": [{ "class": "File", "basename": "foo", "location": "keep:99999999999999999999999999999995+99/bar" }, { "class": "Directory", "basename": "foo2", "location": "keep:99999999999999999999999999999995+99" }, { "class": "File", "basename": "filename", "location": "keep:99999999999999999999999999999995+99/baz/filename" }, { "class": "Directory", "basename": "subdir", "location": "keep:99999999999999999999999999999995+99/subdir" }] }], "baseCommand": "ls" }) make_fs_access = functools.partial( arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_initial_work_dir", make_fs_access=make_fs_access, tmpdir="/tmp"): j.run() call_args, call_kwargs = runner.api.container_requests( ).create.call_args vwdmock.copy.assert_has_calls( [mock.call('bar', 'foo', source_collection=sourcemock)]) vwdmock.copy.assert_has_calls( [mock.call('', 'foo2', source_collection=sourcemock)]) vwdmock.copy.assert_has_calls([ mock.call('baz/filename', 'filename', source_collection=sourcemock) ]) vwdmock.copy.assert_has_calls( [mock.call('subdir', 'subdir', source_collection=sourcemock)]) call_body_expected = { 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_initial_work_dir', 'runtime_constraints': { 'vcpus': 1, 'ram': 1073741824 }, 'use_existing': True, 'priority': 1, 'mounts': { '/tmp': { 'kind': 'tmp', "capacity": 1073741824 }, '/var/spool/cwl': { 'kind': 'tmp', "capacity": 1073741824 }, '/var/spool/cwl/foo': { 'kind': 'collection', 'path': 'foo', 'portable_data_hash': '99999999999999999999999999999996+99' }, '/var/spool/cwl/foo2': { 'kind': 'collection', 'path': 'foo2', 'portable_data_hash': '99999999999999999999999999999996+99' }, '/var/spool/cwl/filename': { 'kind': 'collection', 'path': 'filename', 'portable_data_hash': '99999999999999999999999999999996+99' }, '/var/spool/cwl/subdir': { 'kind': 'collection', 'path': 'subdir', 'portable_data_hash': '99999999999999999999999999999996+99' } }, 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'container_image': 'arvados/jobs', 'command': ['ls'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {} } call_body = call_kwargs.get('body', None) self.assertNotEqual(None, call_body) for key in call_body: self.assertEqual(call_body_expected.get(key), call_body.get(key))
def test_resource_requirements(self, keepdocker): arv_docker_clear_cache() runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } tool = cmap({ "inputs": [], "outputs": [], "hints": [{ "class": "ResourceRequirement", "coresMin": 3, "ramMin": 3000, "tmpdirMin": 4000, "outdirMin": 5000 }, { "class": "http://arvados.org/cwl#RuntimeConstraints", "keep_cache": 512 }, { "class": "http://arvados.org/cwl#APIRequirement", }, { "class": "http://arvados.org/cwl#PartitionRequirement", "partition": "blurb" }], "baseCommand": "ls" }) make_fs_access = functools.partial( arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_resource_requirements", make_fs_access=make_fs_access, tmpdir="/tmp"): j.run() call_args, call_kwargs = runner.api.container_requests( ).create.call_args call_body_expected = { 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_resource_requirements', 'runtime_constraints': { 'vcpus': 3, 'ram': 3145728000, 'keep_cache_ram': 536870912, 'API': True }, 'use_existing': True, 'priority': 1, 'mounts': { '/tmp': { 'kind': 'tmp', "capacity": 4194304000 }, '/var/spool/cwl': { 'kind': 'tmp', "capacity": 5242880000 } }, 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'container_image': 'arvados/jobs', 'command': ['ls'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': { 'partitions': ['blurb'] }, 'properties': {} } call_body = call_kwargs.get('body', None) self.assertNotEqual(None, call_body) for key in call_body: self.assertEqual(call_body_expected.get(key), call_body.get(key))
def test_run(self, list_images_in_arv): for enable_reuse in (True, False): runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]] runner.api.collections().get().execute.return_vaulue = { "portable_data_hash": "99999999999999999999999999999993+99" } tool = cmap({ "inputs": [], "outputs": [], "baseCommand": "ls", "arguments": [{ "valueFrom": "$(runtime.outdir)" }] }) make_fs_access = functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) arvtool = arvados_cwl.ArvadosCommandTool( runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access): j.run(enable_reuse=enable_reuse) runner.api.jobs().create.assert_called_with( body=JsonDiffMatcher({ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'runtime_constraints': {}, 'script_parameters': { 'tasks': [{ 'task.env': { 'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)' }, 'command': ['ls', '$(task.outdir)'] }], }, 'script_version': 'master', 'minimum_script_version': 'a3f2cb186e437bfce0031b024b2157b73ed2717d', 'repository': 'arvados', 'script': 'crunchrunner', 'runtime_constraints': { 'docker_image': 'arvados/jobs', 'min_cores_per_node': 1, 'min_ram_mb_per_node': 1024, 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize } }), find_or_create=enable_reuse, filters=[['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], [ 'script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d' ], [ 'docker_image_locator', 'in docker', 'arvados/jobs' ]])
def test_mounts(self, keepdocker): arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999994+99", "manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2" } document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.1") tool = cmap({ "inputs": [{ "id": "p1", "type": "Directory" }], "outputs": [], "baseCommand": "ls", "arguments": [{ "valueFrom": "$(runtime.outdir)" }], "id": "#", "class": "CommandLineTool" }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_run_mounts" arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) arvtool.formatgraph = None job_order = { "p1": { "class": "Directory", "location": "keep:99999999999999999999999999999994+44", "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "listing": [{ "class": "File", "location": "keep:99999999999999999999999999999994+44/file1", }, { "class": "File", "location": "keep:99999999999999999999999999999994+44/file2", }] } } for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext): j.run(runtimeContext) runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher({ 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_run_mounts', 'runtime_constraints': { 'vcpus': 1, 'ram': 1073741824 }, 'use_existing': True, 'priority': 500, 'mounts': { "/keep/99999999999999999999999999999994+44": { "kind": "collection", "portable_data_hash": "99999999999999999999999999999994+44", "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz" }, '/tmp': { 'kind': 'tmp', "capacity": 1073741824 }, '/var/spool/cwl': { 'kind': 'tmp', "capacity": 1073741824 } }, 'state': 'Committed', 'output_name': 'Output for step test_run_mounts', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, 'container_image': '99999999999999999999999999999994+99', 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {}, 'secret_mounts': {} }))
def test_secrets(self, keepdocker): arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { "portable_data_hash": "99999999999999999999999999999993+99" } document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.1") tool = cmap({ "arguments": ["md5sum", "example.conf"], "class": "CommandLineTool", "hints": [{ "class": "http://commonwl.org/cwltool#Secrets", "secrets": ["#secret_job.cwl/pw"] }], "id": "#secret_job.cwl", "inputs": [{ "id": "#secret_job.cwl/pw", "type": "string" }], "outputs": [], "requirements": [{ "class": "InitialWorkDirRequirement", "listing": [{ "entry": "username: user\npassword: $(inputs.pw)\n", "entryname": "example.conf" }] }] }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_secrets" arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) arvtool.formatgraph = None job_order = {"pw": "blorp"} runner.secret_store.store(["pw"], job_order) for j in arvtool.job(job_order, mock.MagicMock(), runtimeContext): j.run(runtimeContext) runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher({ 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' }, 'name': 'test_secrets', 'runtime_constraints': { 'vcpus': 1, 'ram': 1073741824 }, 'use_existing': True, 'priority': 500, 'mounts': { '/tmp': { 'kind': 'tmp', "capacity": 1073741824 }, '/var/spool/cwl': { 'kind': 'tmp', "capacity": 1073741824 } }, 'state': 'Committed', 'output_name': 'Output for step test_secrets', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, 'container_image': '99999999999999999999999999999993+99', 'command': ['md5sum', 'example.conf'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {}, "secret_mounts": { "/var/spool/cwl/example.conf": { "content": "username: user\npassword: blorp\n", "kind": "text" } } }))
def test_run(self): runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema( "v1.0") tool = { "inputs": [], "outputs": [], "baseCommand": "ls", "arguments": [{ "valueFrom": "$(runtime.outdir)" }] } make_fs_access = functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access): j.run() runner.api.jobs().create.assert_called_with( body={ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'runtime_constraints': {}, 'script_parameters': { 'tasks': [{ 'task.env': { 'HOME': '$(task.outdir)', 'TMPDIR': '$(task.tmpdir)' }, 'command': ['ls', '$(task.outdir)'] }], }, 'script_version': 'master', 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6', 'repository': 'arvados', 'script': 'crunchrunner', 'runtime_constraints': { 'docker_image': 'arvados/jobs', 'min_cores_per_node': 1, 'min_ram_mb_per_node': 1024, 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize } }, find_or_create=True, filters=[['repository', '=', 'arvados'], ['script', '=', 'crunchrunner'], [ 'script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6' ], ['docker_image_locator', 'in docker', 'arvados/jobs']])