def test_handle_stopped_cancelled_task():
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
memory_limit=4,
time_limit=60,
requires_gpu=False,
)
event = {
"taskDefinitionArn": "arn:aws:ecs:region:123456789012:task-definition/algorithms-job-00000000-0000-0000-0000-000000000000:1",
"group": "components-gpu",
"stopCode": "UserInitiated",
"containers": [
{
# No container exit in this case
"name": "algorithms-job-00000000-0000-0000-0000-000000000000-timeout"
},
{
# No container exit in this case
"name": "algorithms-job-00000000-0000-0000-0000-000000000000"
},
],
"createdAt": "2021-09-25T10:50:24.248Z", # No startedAt in this case
"stoppedAt": "2021-09-25T11:02:30.776Z",
}
with pytest.raises(TaskCancelled):
# Task should be retried
executor._get_container_exit_codes(event=event)
def test_handle_stopped_successful_fast_task():
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
memory_limit=4,
time_limit=60,
requires_gpu=False,
)
event = {
"taskDefinitionArn": "arn:aws:ecs:region:123456789012:task-definition/algorithms-job-00000000-0000-0000-0000-000000000000:1",
"group": "components-gpu",
"stopCode": "TaskFailedToStart", # as not all container had time to start
"containers": [
{
# No container exit in this case
"name": "algorithms-job-00000000-0000-0000-0000-000000000000-timeout"
},
{
"exitCode": 0,
"name": "algorithms-job-00000000-0000-0000-0000-000000000000",
},
],
"createdAt": "2021-09-25T10:50:24.248Z", # No startedAt in this case
"stoppedAt": "2021-09-25T11:02:30.776Z",
}
assert executor._get_container_exit_codes(event=event) == {
"algorithms-job-00000000-0000-0000-0000-000000000000": 0
}
def test_ecs_unzip(tmp_path, settings, submission_file):
interface = ComponentInterfaceFactory(
kind=InterfaceKindChoices.ZIP, relative_path="preds.zip"
)
civ = ComponentInterfaceValueFactory(interface=interface)
with open(submission_file, "rb") as f:
civ.file.save("my_submission.zip", File(f))
settings.COMPONENTS_AMAZON_ECS_NFS_MOUNT_POINT = tmp_path
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
memory_limit=4,
time_limit=60,
requires_gpu=False,
)
executor.provision(input_civs=[civ], input_prefixes={})
assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == {
"algorithms",
"algorithms/job",
"algorithms/job/00000000-0000-0000-0000-000000000000",
"algorithms/job/00000000-0000-0000-0000-000000000000/input",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/submission.csv",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images/image10x10x10.mhd",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images/image10x10x10.zraw",
"algorithms/job/00000000-0000-0000-0000-000000000000/output",
}
def test_input_prefixes(tmp_path, settings):
interfaces = [
ComponentInterfaceFactory(
kind=InterfaceKindChoices.BOOL, relative_path="test/bool.json"
),
ComponentInterfaceFactory(
kind=InterfaceKindChoices.IMAGE, relative_path="images/test-image"
),
ComponentInterfaceFactory(
kind=InterfaceKindChoices.CSV, relative_path="test.csv"
),
]
civs = [
ComponentInterfaceValueFactory(interface=interfaces[0], value=True),
ComponentInterfaceValueFactory(
interface=interfaces[1],
image=ImageFileFactory(
file__from_path=Path(__file__).parent.parent
/ "algorithms_tests"
/ "resources"
/ "input_file.tif"
).image,
),
ComponentInterfaceValueFactory(interface=interfaces[2]),
]
settings.COMPONENTS_AMAZON_ECS_NFS_MOUNT_POINT = tmp_path
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
memory_limit=4,
time_limit=60,
requires_gpu=False,
)
executor.provision(
input_civs=civs,
input_prefixes={
str(civs[0].pk): "first/output/",
str(civs[1].pk): "second/output",
},
)
assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == {
"algorithms",
"algorithms/job",
"algorithms/job/00000000-0000-0000-0000-000000000000",
"algorithms/job/00000000-0000-0000-0000-000000000000/input",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/test.csv",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/first",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/first/output",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/first/output/test",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/first/output/test/bool.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output/images",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output/images/test-image",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output/images/test-image/input_file.tif",
"algorithms/job/00000000-0000-0000-0000-000000000000/output",
}
def test_set_duration_fast_task():
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
exec_image_file=None,
memory_limit=4,
requires_gpu=False,
)
executor._set_duration(
event={
"createdAt": "2021-09-25T10:50:24.248Z",
"stoppedAt": "2021-09-25T11:02:30.776Z",
})
assert executor.duration == datetime.timedelta(seconds=726,
microseconds=528000)
def test_handle_stopped_successful_task():
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
exec_image_file=None,
memory_limit=4,
requires_gpu=False,
)
event = {
"taskDefinitionArn":
"arn:aws:ecs:region:123456789012:task-definition/algorithms-job-00000000-0000-0000-0000-000000000000:1",
"group":
"components-gpu",
"stopCode":
"EssentialContainerExited",
"containers": [
{
"exitCode":
143,
"name":
"algorithms-job-00000000-0000-0000-0000-000000000000-timeout",
},
{
"exitCode": 0,
"name": "algorithms-job-00000000-0000-0000-0000-000000000000",
},
],
"startedAt":
"2021-09-25T10:50:24.248Z",
"stoppedAt":
"2021-09-25T11:02:30.776Z",
}
assert executor._get_container_exit_codes(event=event) == {
"algorithms-job-00000000-0000-0000-0000-000000000000": 0,
"algorithms-job-00000000-0000-0000-0000-000000000000-timeout": 143,
}
def test_provision(tmp_path, settings):
interfaces = [
ComponentInterfaceFactory(
kind=InterfaceKindChoices.BOOL, relative_path="test/bool.json"
),
ComponentInterfaceFactory(
kind=InterfaceKindChoices.IMAGE, relative_path="images/test-image"
),
ComponentInterfaceFactory(
kind=InterfaceKindChoices.CSV, relative_path="test.csv"
),
]
civs = [
ComponentInterfaceValueFactory(interface=interfaces[0], value=True),
ComponentInterfaceValueFactory(
interface=interfaces[1],
image=ImageFileFactory(
file__from_path=Path(__file__).parent.parent
/ "algorithms_tests"
/ "resources"
/ "input_file.tif"
).image,
),
ComponentInterfaceValueFactory(interface=interfaces[2]),
]
civs[2].file.save("whatever.csv", ContentFile(b"foo,\nbar,\n"))
settings.COMPONENTS_AMAZON_ECS_NFS_MOUNT_POINT = tmp_path
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
memory_limit=4,
time_limit=60,
requires_gpu=False,
)
executor.provision(input_civs=civs, input_prefixes={})
executor.execute()
executor.handle_event(
event={
# Minimal successful event
"taskDefinitionArn": "arn:aws:ecs:region:123456789012:task-definition/algorithms-job-00000000-0000-0000-0000-000000000000:1",
"group": "components-gpu",
"stopCode": "EssentialContainerExited",
"containers": [
{
"exitCode": 143,
"name": "algorithms-job-00000000-0000-0000-0000-000000000000-timeout",
},
{
"exitCode": 0,
"name": "algorithms-job-00000000-0000-0000-0000-000000000000",
},
],
"startedAt": "2021-09-25T10:50:24.248Z",
"stoppedAt": "2021-09-25T11:02:30.776Z",
}
)
assert executor.duration == datetime.timedelta(
seconds=726, microseconds=528000
)
assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == {
"algorithms",
"algorithms/job",
"algorithms/job/00000000-0000-0000-0000-000000000000",
"algorithms/job/00000000-0000-0000-0000-000000000000/input",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/test.csv",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/test",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/test/bool.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images/test-image",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images/test-image/input_file.tif",
"algorithms/job/00000000-0000-0000-0000-000000000000/output",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/metrics.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/results.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/test.csv",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/test",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/test/bool.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/images",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/images/test-image",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/images/test-image/input_file.tif",
}
# Exclude the CIV reading as this is unsupported
outputs = executor.get_outputs(output_interfaces=interfaces)
assert len(outputs) == 3
executor.deprovision()
assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == {
"algorithms",
"algorithms/job",
}