def test_create_executor_output_streams(self, app, mocker):
"""
Verifies presence of output stream mapping in executor causes adds an
additional task to handle stream copy
"""
mocked_batch_client, mocked_blob_client, mocked_file_client = setup_common_mocks_for_create(
mocker)
task = models.TesTask(
name="task-name",
description="task-description",
executors=[
models.TesExecutor(
image="ubuntu:latest",
command=["ls", "-l"],
stdout="/tes-wd/shared/executions/stdout.txt",
stderr="/tes-wd/shared/executions/stderr.txt"),
models.TesExecutor(
image="ubuntu:latest",
command=["ls", "-l"],
stdout="/tes-wd/shared/executions/stdout.txt",
stderr="/tes-wd/shared/executions/stderr.txt")
])
result = compute_backend.backend.createJob(task)
assert (uuid.UUID(result))
assert (mocked_batch_client.return_value.job.add.call_count == 1)
assert (mocked_batch_client.return_value.task.add.call_count == len(
task.executors))
def test_multiple_executors(self, app, mocker):
"""
Verifies task is added for each executor
"""
mocked_batch_client, mocked_blob_client, mocked_file_client = setup_common_mocks_for_create(
mocker)
task = models.TesTask(
name="task-name",
description="task-description",
executors=[
models.TesExecutor(image="alpine", command=["pwd"]),
models.TesExecutor(
image="ubuntu:latest",
command=["ls", "-l"],
workdir="/tes-wd/shared",
),
models.TesExecutor(
image=
"ubuntu@sha256:868fd30a0e47b8d8ac485df174795b5e2fe8a6c8f056cc707b232d65b8a1ab68",
command=["ls -l"],
workdir="/tes-wd",
)
])
result = compute_backend.backend.createJob(task)
assert (uuid.UUID(result))
assert (mocked_batch_client.return_value.job.add.call_count == 1)
assert (mocked_batch_client.return_value.task.add.call_count == len(
task.executors))
def test_create_executor_environment(self, app, mocker):
"""
Verifies environment variables are mapped to tasks
"""
environ = {"foo": "bar"}
mocked_batch_client, mocked_blob_client, mocked_file_client = setup_common_mocks_for_create(
mocker)
task = models.TesTask(name="task-name",
description="task-description",
executors=[
models.TesExecutor(
image="ubuntu:latest",
command=["ls", "-l"],
env=environ,
)
])
result = compute_backend.backend.createJob(task)
assert (uuid.UUID(result))
# ensure env mappings are present
args, kwargs = mocked_batch_client.return_value.task.add.call_args
batch_task = kwargs['task']
assert (isinstance(batch_task, azbatch.models.TaskAddParameter))
assert (batch_task.environment_settings == [
azbatch.models.EnvironmentSetting(name=k, value=v)
for k, v in environ.items()
])
def test_create_output_url_no_input(self, app, mocker):
"""
Verifies stub prep task is used when outputs are present without inputs
"""
mocked_batch_client, mocked_blob_client, mocked_file_client = setup_common_mocks_for_create(
mocker)
task = models.TesTask(
name="task-name",
description="task-description",
outputs=[
models.TesOutput(
url=
"https://tesazure.blob.core.windows.net/samples/random.dat",
path="random.dat",
description="output-description",
name="output-name",
type=models.TesFileType.FILE)
])
result = compute_backend.backend.createJob(task)
assert (uuid.UUID(result))
assert (
mocked_blob_client.return_value.create_container.call_count == 1)
assert (mocked_batch_client.return_value.job.add.call_count == 1)
args, kwargs = mocked_batch_client.return_value.job.add.call_args
batch_job = args[0]
assert ("true" in batch_job.job_preparation_task.command_line)
assert ("pipefail" in batch_job.job_release_task.command_line)
def test_create_input_content(self, app, mocker):
"""
Verifies temporary blob is created & corresponding prep task added when
URL input is provided
"""
mocked_batch_client, mocked_blob_client, mocked_file_client = setup_common_mocks_for_create(
mocker)
mocked_blob_client.return_value.make_blob_url.side_effect = [
"https://account.blob.core.windows.net/container/blob"
]
task = models.TesTask(name="task-name",
description="task-description",
inputs=[
models.TesInput(
path="/tes-wd/shared/script",
description="Should echo OK",
content='#!/bin/bash\necho "OK"',
name="commandScript",
type=models.TesFileType.FILE)
])
result = compute_backend.backend.createJob(task)
assert (uuid.UUID(result))
assert (
mocked_blob_client.return_value.create_container.call_count == 1)
assert (mocked_blob_client.return_value.create_blob_from_text.
call_count == 1)
assert (mocked_batch_client.return_value.job.add.call_count == 1)
args, kwargs = mocked_batch_client.return_value.job.add.call_args
job_preparation_task = args[0].job_preparation_task
assert ("pipefail" in job_preparation_task.command_line)
def test_create_input_url(self, app, mocker):
"""
Verifies prep task is added without using temporary blob when URL input
is provided
"""
mocked_batch_client, mocked_blob_client, mocked_file_client = setup_common_mocks_for_create(
mocker)
task = models.TesTask(
name="task-name",
description="task-description",
inputs=[
models.TesInput(
url=
"https://tesazure.blob.core.windows.net/samples/random.dat",
path="random.dat",
description="input-description",
name="input-name",
type=models.TesFileType.FILE,
)
])
result = compute_backend.backend.createJob(task)
assert (uuid.UUID(result))
assert (
mocked_blob_client.return_value.create_container.call_count == 1)
assert (mocked_blob_client.return_value.create_blob_from_text.
call_count == 0)
assert (mocked_batch_client.return_value.job.add.call_count == 1)
args, kwargs = mocked_batch_client.return_value.job.add.call_args
job_preparation_task = args[0].job_preparation_task
assert ("pipefail" in job_preparation_task.command_line)
def test_create_no_marshalling(self, app, mocker):
"""
Verifies no prep/release tasks are used if not inputs or outputs are
provided
"""
mocked_batch_client, mocked_blob_client, mocked_file_client = setup_common_mocks_for_create(
mocker)
task = models.TesTask(name="task-name",
description="task-description",
inputs=[],
outputs=[])
result = compute_backend.backend.createJob(task)
assert (uuid.UUID(result))
assert (mocked_batch_client.return_value.job.add.call_count == 1)
args, kwargs = mocked_batch_client.return_value.job.add.call_args
batch_job = args[0]
assert ("true" in batch_job.job_preparation_task.command_line)
assert (batch_job.job_release_task is None)
def test_create_job_exists(self, app, mocker):
"""
Verifies failure when job ID already exists in Batch account
"""
hardcoded_uuid = 'b95e3451-5cd0-4e46-b595-6e8b0bb9bb62'
returned_uuids = [uuid.UUID(hardcoded_uuid)]
mocked_batch_client, mocked_blob_client, mocked_file_client = setup_common_mocks_for_create(
mocker)
mocked_batch_client.return_value.job.list.return_value = [
azbatch.models.CloudJob(id=hardcoded_uuid)
]
mock_uuid = mocker.patch('tesazure.backends.batch.uuid.uuid4',
autospec=True)
mock_uuid.side_effect = returned_uuids
task = models.TesTask()
result = compute_backend.backend.createJob(task)
assert (result is False)
assert (
mocked_blob_client.return_value.create_container.call_count == 0)
assert (mocked_batch_client.return_value.job.add.call_count == 0)
def test_task_mangling_cromwell(self, mocker):
mocked_blob_client = mocker.patch('azure.storage.blob.BlockBlobService')
task = tesmodels.TesTask(
inputs=[
tesmodels.TesInput(name='outside_prefix', path='/foo', url='https://other-source', type=tesmodels.TesFileType.FILE),
tesmodels.TesInput(name='within_prefix-nested', path='/tes-wd/shared/foo/bar', url='/tes-wd/shared/foo/bar', type=tesmodels.TesFileType.FILE),
tesmodels.TesInput(name='within_prefix-global', path='/tes-wd/shared-global/foo/bar', url='/tes-wd/shared-global/foo/bar', type=tesmodels.TesFileType.FILE),
tesmodels.TesInput(name='external_source', path='/tes-wd/shared/other/source', url='https://other-source', type=tesmodels.TesFileType.FILE),
tesmodels.TesInput(name='external_source_global', path='/tes-wd/shared-global/other/source', url='https://other-source', type=tesmodels.TesFileType.FILE),
tesmodels.TesInput(name='raw_content', path='/tes-wd/shared/raw', content='raw_content', type=tesmodels.TesFileType.FILE),
],
outputs=[
tesmodels.TesOutput(name='outside_prefix', path='/tes-wd/foo', url='/tes-wd/foo', type=tesmodels.TesFileType.FILE),
tesmodels.TesOutput(name='within_prefix-nested', path='/tes-wd/shared/foo/bar', url='/tes-wd/shared/foo/bar', type=tesmodels.TesFileType.FILE),
tesmodels.TesOutput(name='within_prefix-global', path='/tes-wd/shared-global/foo/bar', url='/tes-wd/shared-global/foo/bar', type=tesmodels.TesFileType.FILE),
tesmodels.TesOutput(name='external_source', path='/tes-wd/shared-global/other/source', url='https://other-source', type=tesmodels.TesFileType.FILE),
],
tags={
'ms-submitter-name': 'cromwell',
'ms-submitter-workflow-id': '0911c1c7-de5d-442f-94e8-31814a035f8c',
'ms-submitter-workflow-name': 'wf_hello',
'ms-submitter-workflow-stage': 'hello',
'ms-submitter-cromwell-executiondir': '/tes-wd/shared/wf_hello/0911c1c7-de5d-442f-94e8-31814a035f8c/call-hello/execution'
}
)
mocked_blob_client.return_value.make_blob_url.return_value = "http://account.blob.core.windows.net/foo/bar"
mocked_blob_client.return_value.list_blob_names.return_value = [
task.tags.get('ms-submitter-cromwell-executiondir') + '/foo', # this is will get uploaded (since input 'foo' above is not from cromwell)
task.tags.get('ms-submitter-cromwell-executiondir') + '/bar', # this is will get ignored (since input '/tes-wd/shared/bar' above already exists from cromwell)
task.tags.get('ms-submitter-cromwell-executiondir') + '/write_tsv' # this is will get uploaded (since it matches no cromwell inputs above)
]
orig_num_inputs = len(task.inputs)
common.mangle_task_for_submitter(task)
assert(mocked_blob_client.return_value.create_container.call_count == 1)
assert(mocked_blob_client.return_value.list_blob_names.call_count == 1)
assert(mocked_blob_client.return_value.generate_container_shared_access_signature.call_count == 2)
# Two for mangled inputs, two for mangled output, two for injected
assert(mocked_blob_client.return_value.make_blob_url.call_count == 2 + 2 + 2)
# outside prefix should not be mangled, even if it otherwise matches
assert(task.inputs[0].name == 'outside_prefix' and task.inputs[0].path == '/foo' and task.inputs[0].url == 'https://other-source')
assert(task.outputs[0].name == 'outside_prefix' and task.outputs[0].path == task.outputs[0].url)
# within prefix should be mangled
assert(task.inputs[1].name == 'within_prefix-nested' and task.inputs[1].path != task.inputs[1].url)
assert(task.inputs[2].name == 'within_prefix-global' and task.inputs[2].path != task.inputs[2].url)
assert(task.outputs[1].name == 'within_prefix-nested' and task.outputs[1].path != task.outputs[1].url)
assert(task.outputs[2].name == 'within_prefix-global' and task.outputs[2].path != task.outputs[2].url)
# external sources should not be modified
assert(task.inputs[3].name == 'external_source' and task.inputs[3].path == '/tes-wd/shared/other/source' and task.inputs[3].url == 'https://other-source')
assert(task.inputs[4].name == 'external_source_global' and task.inputs[4].path == '/tes-wd/shared-global/other/source' and task.inputs[4].url == 'https://other-source')
assert(task.outputs[3].name == 'external_source' and task.outputs[3].url == 'https://other-source')
# raw content should be ignored
assert(task.inputs[5].name == 'raw_content' and not task.inputs[5].url)
# raw content should be ignored
assert(task.inputs[5].name == 'raw_content' and not task.inputs[5].url)
# injected
assert(len(task.inputs) == orig_num_inputs + 2)
assert(task.inputs[-2].path == task.tags.get('ms-submitter-cromwell-executiondir') + '/foo')
assert(task.inputs[-1].path == task.tags.get('ms-submitter-cromwell-executiondir') + '/write_tsv')
def test_tes_task_procedural_build(self, session):
schema = models.TesTaskSchema()
# Read a sample task from file
with open(
os.path.join('tests', 'unit', 'data',
'test_models_task.json')) as fh:
task_json = json.loads(fh.read())
task_from_file = schema.load(task_json).data
# Define same task programattically
tags = {"tag-key": "tag-value"}
executors = [
models.TesExecutor(image="alpine", command=["pwd"]),
models.TesExecutor(image="ubuntu:latest",
command=["ls", "-l"],
env={"foo": "bar"},
workdir="/tes-wd/shared",
stdout="/tes-wd/shared/executions/stdout.txt",
stderr="/tes-wd/shared/executions/stderr.txt"),
models.TesExecutor(
image=
"ubuntu@sha256:868fd30a0e47b8d8ac485df174795b5e2fe8a6c8f056cc707b232d65b8a1ab68",
command=["cat"],
workdir="/tes-wd/shared",
stdin="/tes-wd/shared/executions/stdin.txt")
]
resources = models.TesResources(cpu_cores=4,
disk_gb=4,
preemptible=True,
ram_gb=7)
inputs = [
models.TesInput(
url="https://tesazure.blob.core.windows.net/samples/random.dat",
path="random.dat",
description="input-description",
name="input-name",
type=models.TesFileType.FILE,
),
models.TesInput(path="/tes-wd/shared/script",
description="Should echo OK",
content='#!/bin/bash\necho "OK"',
name="commandScript",
type=models.TesFileType.FILE)
]
outputs = [
models.TesOutput(
url="https://tesazure.blob.core.windows.net/samples/random.dat",
path="random.dat",
description="output-description",
name="output-name",
type=models.TesFileType.FILE)
]
task_from_code = models.TesTask(name="task-name",
description="task-description",
tags=tags,
executors=executors,
resources=resources,
inputs=inputs,
outputs=outputs)
# Ensure they are equivalent
assert (schema.dump(task_from_code).data == schema.dump(
task_from_file).data)